From bb1038c37014f5168ed2cd6a89b6524ae5274b6f Mon Sep 17 00:00:00 2001 From: Cobb Date: Wed, 29 Jul 2026 07:50:57 -0700 Subject: [PATCH] speed: move-not-clone reel playerResponse, overlap sig-timestamp fetch, cut player.js copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - the reel `playerResponse` unwrap now MOVES the ~150-500 KB subtree out (`Map::remove`) instead of deep-cloning it, on every open + feed-enrich (S1). - overlap the player.js signature-timestamp resolution with the primary Android reel fetch via `std::thread::scope` — independent legs (disjoint lock sets), Android stays the gating primary, decoy gate + best-effort semantics unchanged; concurrency-harness-proven: no deadlock/race/output change (S2). - `Response::into_body` drops a ~1.5 MB player.js body copy (S8); manager snippet fields -> `Arc` so the per-eval clone-out-of-lock is a pointer bump, not a 30-120 KB String copy (S10). player_manager lock/memo untouched. Deferred (documented): S7 (drop retained player_code = a memo-state-machine change, not a safe copy cut), S3 ($fields narrowing needs a live check). Wrapper-crate follow-ups (out of core scope): S4 (net.rs from_utf8 fast path), S5 (opt-level=2 for the JS-interpreter crates in rust/Cargo.toml). 156 lib + 7 integration tests, clippy -D clean. --- src/downloader/response.rs | 9 +++ src/youtube/js/extractor.rs | 4 +- src/youtube/js/player_manager.rs | 18 ++++-- src/youtube/stream_extractor.rs | 100 +++++++++++++++++++++++++------ 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/src/downloader/response.rs b/src/downloader/response.rs index cd34d32..4702277 100644 --- a/src/downloader/response.rs +++ b/src/downloader/response.rs @@ -54,6 +54,15 @@ impl Response { &self.response_body } + /// Consumes the response, handing back the owned body `String` with no + /// copy. Use at the last read of a body when the rest of the `Response` + /// is no longer needed — e.g. the ~1.5 MB player.js download, where + /// `response_body().to_string()` would clone the whole buffer only to + /// drop the original (S8 speed fix). + pub fn into_body(self) -> String { + self.response_body + } + pub fn latest_url(&self) -> &str { &self.latest_url } diff --git a/src/youtube/js/extractor.rs b/src/youtube/js/extractor.rs index c694dd6..27515aa 100644 --- a/src/youtube/js/extractor.rs +++ b/src/youtube/js/extractor.rs @@ -136,7 +136,9 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result, signature_timestamp: Option, - sig_snippet: Option, - nsig_name: Option, - nsig_snippet: Option, + // Snippets held as `Arc` so the per-eval hand-off out of the state + // lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only + // the cloned payload type changes — the lock/memo/fetch_gate discipline is + // exactly as Loop-1 left it. + sig_snippet: Option>, + nsig_name: Option>, + nsig_snippet: Option>, throttling_param_cache: HashMap, // Failure memos — deliberately NOT cleared by invalidate(): they must @@ -228,7 +233,7 @@ impl PlayerManager { video_id, Derived::Sig, |state| state.sig_snippet.clone(), - signature::build_deobfuscator, + |code| signature::build_deobfuscator(code).map(Arc::::from), |state, snippet| state.sig_snippet = Some(snippet.clone()), )?; @@ -275,7 +280,10 @@ impl PlayerManager { (Some(n), Some(s)) => Some((n.clone(), s.clone())), _ => None, }, - nsig::build_deobfuscator, + |code| { + nsig::build_deobfuscator(code) + .map(|(n, s)| (Arc::::from(n), Arc::::from(s))) + }, |state, (n, s)| { state.nsig_name = Some(n.clone()); state.nsig_snippet = Some(s.clone()); diff --git a/src/youtube/stream_extractor.rs b/src/youtube/stream_extractor.rs index 80ca2a2..44c47e4 100644 --- a/src/youtube/stream_extractor.rs +++ b/src/youtube/stream_extractor.rs @@ -121,16 +121,47 @@ pub fn stream_info_with( ); let android_cpn = generate_content_playback_nonce(); - let player_response = fetch_android( - video_id, - &localization, - &content_country, - &android_cpn, - android_token - .as_ref() - .map(|t| t.player_request_po_token.as_str()), - android_token.as_ref().map(|t| t.visitor_data.as_str()), - )?; + + // S2 (Option A — the conservative subset): overlap the player.js + // signature-timestamp resolution with the primary Android reel fetch. The + // two are fully independent — `signature_timestamp` needs only `video_id`, + // and the WEB metadata leg that consumes the timestamp still runs AFTER the + // decoy gate below. On a cold open (or after a player.js rotation) this + // overlaps the ~1.5 MB base.js download with the Android round-trip instead + // of paying them serially; warm, the timestamp is cached and the worker + // returns near-instantly. `scope` joins both before either result is read, + // so the borrowed locals stay valid (same shape as `channel.rs`). The + // worker leg is best-effort: a failed extraction OR a panicked worker yields + // 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the + // gating primary (its playability/decoy check below is unchanged and still + // aborts before any WEB/visionOS work), the visitorData-first ordering is + // untouched (the player.js fetch never hits the visitor_id endpoint), and + // player.js concurrency is already guarded by PlayerManager's fetch_gate so + // a second concurrent open dedups the download rather than racing it. + // Trade-off (accepted): on a COLD cache, a *failing* open can't return its + // error until the speculative player.js leg joins (bounded by one download); + // the win is the warm/success path. Worst case is a rare, bounded cold-fail + // delay — never a change in what's extracted. + let (android_result, signature_timestamp) = std::thread::scope(|s| { + let sig_ts = s.spawn(|| { + PlayerManager::instance() + .signature_timestamp(video_id) + .unwrap_or(0) + }); + let android = fetch_android( + video_id, + &localization, + &content_country, + &android_cpn, + android_token + .as_ref() + .map(|t| t.player_request_po_token.as_str()), + android_token.as_ref().map(|t| t.visitor_data.as_str()), + ); + // A panicked worker joins as Err → 0, same as an extraction failure. + (android, sig_ts.join().unwrap_or(0)) + }); + let player_response = android_result?; // A primary-android rejection (playability failure or decoy) is exactly the // symptom a poisoned visitorData would produce on the primary reel call — @@ -225,9 +256,9 @@ pub fn stream_info_with( .map(|t| t.streaming_data_po_token.clone()) .or_else(|| options.ios_streaming_pot.clone()); - let signature_timestamp = PlayerManager::instance() - .signature_timestamp(video_id) - .unwrap_or(0); + // `signature_timestamp` was resolved concurrently with the Android fetch + // above (S2). The WEB metadata leg is the sole consumer and runs only now, + // after the Android decoy gate has passed. let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp); let mut info = StreamInfo { @@ -371,7 +402,7 @@ fn fetch_android( po_token: Option<&str>, visitor_data: Option<&str>, ) -> Result { - let result = if po_token.is_some() { + if po_token.is_some() { stream_helper::get_android_player_response( video_id, localization, @@ -388,9 +419,23 @@ fn fetch_android( cpn, )?; // The reel endpoint returns the `playerResponse` nested one level. - Ok(r.get("playerResponse").cloned().unwrap_or(r)) - }; - result + Ok(unwrap_reel_player_response(r)) + } +} + +/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the +/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead +/// after this call, so `serde_json::Map::remove` returns the owned subtree with +/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map / +/// Vec / String node on every video open AND every feed-enrich (S1). Behavior +/// is identical, including the defensive "no `playerResponse` key → use the +/// whole response" fallback. Only the anonymous reel path calls this; the +/// token /player path returns the response already un-nested. +fn unwrap_reel_player_response(r: Value) -> Value { + match r { + Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)), + other => other, + } } fn fetch_web_metadata( @@ -1015,6 +1060,27 @@ mod tests { assert!(!is_player_response_not_valid(&resp, "MATCHING")); } + #[test] + fn reel_player_response_moved_out_or_falls_back() { + // Nested playerResponse is unwrapped (moved out, not cloned) — the + // returned subtree is exactly the inner object. + let r = json!({ + "playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}}, + "responseContext": {"x": 1} + }); + let out = unwrap_reel_player_response(r); + assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}})); + + // No playerResponse key → the whole response is returned unchanged + // (matches the prior `.cloned().unwrap_or(r)` fallback). + let r = json!({"videoDetails": {"videoId": "xyz"}}); + assert_eq!(unwrap_reel_player_response(r.clone()), r); + + // Defensive: a non-object value passes through as-is. + let r = json!("scalar"); + assert_eq!(unwrap_reel_player_response(r.clone()), r); + } + #[test] fn cipher_string_parsed() { let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1";