speed: move-not-clone reel playerResponse, overlap sig-timestamp fetch, cut player.js copies
Some checks failed
gitleaks / scan (push) Failing after 13m8s

- the reel `playerResponse` unwrap now MOVES the ~150-500 KB subtree out
  (`Map::remove`) instead of deep-cloning it, on every open + feed-enrich (S1).
- overlap the player.js signature-timestamp resolution with the primary Android
  reel fetch via `std:🧵:scope` — independent legs (disjoint lock sets),
  Android stays the gating primary, decoy gate + best-effort semantics
  unchanged; concurrency-harness-proven: no deadlock/race/output change (S2).
- `Response::into_body` drops a ~1.5 MB player.js body copy (S8); manager
  snippet fields -> `Arc<str>` so the per-eval clone-out-of-lock is a pointer
  bump, not a 30-120 KB String copy (S10). player_manager lock/memo untouched.

Deferred (documented): S7 (drop retained player_code = a memo-state-machine
change, not a safe copy cut), S3 ($fields narrowing needs a live check).
Wrapper-crate follow-ups (out of core scope): S4 (net.rs from_utf8 fast path),
S5 (opt-level=2 for the JS-interpreter crates in rust/Cargo.toml).
156 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
Cobb 2026-07-29 07:50:57 -07:00
parent b5dde59464
commit bb1038c370
4 changed files with 108 additions and 23 deletions

View file

@ -54,6 +54,15 @@ impl Response {
&self.response_body
}
/// Consumes the response, handing back the owned body `String` with no
/// copy. Use at the last read of a body when the rest of the `Response`
/// is no longer needed — e.g. the ~1.5 MB player.js download, where
/// `response_body().to_string()` would clone the whole buffer only to
/// drop the original (S8 speed fix).
pub fn into_body(self) -> String {
self.response_body
}
pub fn latest_url(&self) -> &str {
&self.latest_url
}

View file

@ -136,7 +136,9 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result<St
resp.response_code()
)));
}
Ok(resp.response_body().to_string())
// Consume the owned body — no second ~1.5 MB copy (S8). `resp` is dead
// after the response-code check above.
Ok(resp.into_body())
}
#[cfg(test)]

View file

@ -51,6 +51,7 @@
use parking_lot::Mutex;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::youtube::js::extractor;
@ -155,9 +156,13 @@ struct ManagerState {
player_code: Option<String>,
signature_timestamp: Option<i32>,
sig_snippet: Option<String>,
nsig_name: Option<String>,
nsig_snippet: Option<String>,
// Snippets held as `Arc<str>` so the per-eval hand-off out of the state
// lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only
// the cloned payload type changes — the lock/memo/fetch_gate discipline is
// exactly as Loop-1 left it.
sig_snippet: Option<Arc<str>>,
nsig_name: Option<Arc<str>>,
nsig_snippet: Option<Arc<str>>,
throttling_param_cache: HashMap<String, String>,
// Failure memos — deliberately NOT cleared by invalidate(): they must
@ -228,7 +233,7 @@ impl PlayerManager {
video_id,
Derived::Sig,
|state| state.sig_snippet.clone(),
signature::build_deobfuscator,
|code| signature::build_deobfuscator(code).map(Arc::<str>::from),
|state, snippet| state.sig_snippet = Some(snippet.clone()),
)?;
@ -275,7 +280,10 @@ impl PlayerManager {
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
_ => None,
},
nsig::build_deobfuscator,
|code| {
nsig::build_deobfuscator(code)
.map(|(n, s)| (Arc::<str>::from(n), Arc::<str>::from(s)))
},
|state, (n, s)| {
state.nsig_name = Some(n.clone());
state.nsig_snippet = Some(s.clone());

View file

@ -121,16 +121,47 @@ pub fn stream_info_with(
);
let android_cpn = generate_content_playback_nonce();
let player_response = fetch_android(
video_id,
&localization,
&content_country,
&android_cpn,
android_token
.as_ref()
.map(|t| t.player_request_po_token.as_str()),
android_token.as_ref().map(|t| t.visitor_data.as_str()),
)?;
// S2 (Option A — the conservative subset): overlap the player.js
// signature-timestamp resolution with the primary Android reel fetch. The
// two are fully independent — `signature_timestamp` needs only `video_id`,
// and the WEB metadata leg that consumes the timestamp still runs AFTER the
// decoy gate below. On a cold open (or after a player.js rotation) this
// overlaps the ~1.5 MB base.js download with the Android round-trip instead
// of paying them serially; warm, the timestamp is cached and the worker
// returns near-instantly. `scope` joins both before either result is read,
// so the borrowed locals stay valid (same shape as `channel.rs`). The
// worker leg is best-effort: a failed extraction OR a panicked worker yields
// 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the
// gating primary (its playability/decoy check below is unchanged and still
// aborts before any WEB/visionOS work), the visitorData-first ordering is
// untouched (the player.js fetch never hits the visitor_id endpoint), and
// player.js concurrency is already guarded by PlayerManager's fetch_gate so
// a second concurrent open dedups the download rather than racing it.
// Trade-off (accepted): on a COLD cache, a *failing* open can't return its
// error until the speculative player.js leg joins (bounded by one download);
// the win is the warm/success path. Worst case is a rare, bounded cold-fail
// delay — never a change in what's extracted.
let (android_result, signature_timestamp) = std::thread::scope(|s| {
let sig_ts = s.spawn(|| {
PlayerManager::instance()
.signature_timestamp(video_id)
.unwrap_or(0)
});
let android = fetch_android(
video_id,
&localization,
&content_country,
&android_cpn,
android_token
.as_ref()
.map(|t| t.player_request_po_token.as_str()),
android_token.as_ref().map(|t| t.visitor_data.as_str()),
);
// A panicked worker joins as Err → 0, same as an extraction failure.
(android, sig_ts.join().unwrap_or(0))
});
let player_response = android_result?;
// A primary-android rejection (playability failure or decoy) is exactly the
// symptom a poisoned visitorData would produce on the primary reel call —
@ -225,9 +256,9 @@ pub fn stream_info_with(
.map(|t| t.streaming_data_po_token.clone())
.or_else(|| options.ios_streaming_pot.clone());
let signature_timestamp = PlayerManager::instance()
.signature_timestamp(video_id)
.unwrap_or(0);
// `signature_timestamp` was resolved concurrently with the Android fetch
// above (S2). The WEB metadata leg is the sole consumer and runs only now,
// after the Android decoy gate has passed.
let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp);
let mut info = StreamInfo {
@ -371,7 +402,7 @@ fn fetch_android(
po_token: Option<&str>,
visitor_data: Option<&str>,
) -> Result<Value, ExtractionError> {
let result = if po_token.is_some() {
if po_token.is_some() {
stream_helper::get_android_player_response(
video_id,
localization,
@ -388,9 +419,23 @@ fn fetch_android(
cpn,
)?;
// The reel endpoint returns the `playerResponse` nested one level.
Ok(r.get("playerResponse").cloned().unwrap_or(r))
};
result
Ok(unwrap_reel_player_response(r))
}
}
/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the
/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead
/// after this call, so `serde_json::Map::remove` returns the owned subtree with
/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map /
/// Vec / String node on every video open AND every feed-enrich (S1). Behavior
/// is identical, including the defensive "no `playerResponse` key → use the
/// whole response" fallback. Only the anonymous reel path calls this; the
/// token /player path returns the response already un-nested.
fn unwrap_reel_player_response(r: Value) -> Value {
match r {
Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)),
other => other,
}
}
fn fetch_web_metadata(
@ -1015,6 +1060,27 @@ mod tests {
assert!(!is_player_response_not_valid(&resp, "MATCHING"));
}
#[test]
fn reel_player_response_moved_out_or_falls_back() {
// Nested playerResponse is unwrapped (moved out, not cloned) — the
// returned subtree is exactly the inner object.
let r = json!({
"playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}},
"responseContext": {"x": 1}
});
let out = unwrap_reel_player_response(r);
assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}}));
// No playerResponse key → the whole response is returned unchanged
// (matches the prior `.cloned().unwrap_or(r)` fallback).
let r = json!({"videoDetails": {"videoId": "xyz"}});
assert_eq!(unwrap_reel_player_response(r.clone()), r);
// Defensive: a non-object value passes through as-is.
let r = json!("scalar");
assert_eq!(unwrap_reel_player_response(r.clone()), r);
}
#[test]
fn cipher_string_parsed() {
let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1";