speed: move-not-clone reel playerResponse, overlap sig-timestamp fetch, cut player.js copies
Some checks failed
gitleaks / scan (push) Failing after 13m8s
Some checks failed
gitleaks / scan (push) Failing after 13m8s
- the reel `playerResponse` unwrap now MOVES the ~150-500 KB subtree out
(`Map::remove`) instead of deep-cloning it, on every open + feed-enrich (S1).
- overlap the player.js signature-timestamp resolution with the primary Android
reel fetch via `std:🧵:scope` — independent legs (disjoint lock sets),
Android stays the gating primary, decoy gate + best-effort semantics
unchanged; concurrency-harness-proven: no deadlock/race/output change (S2).
- `Response::into_body` drops a ~1.5 MB player.js body copy (S8); manager
snippet fields -> `Arc<str>` so the per-eval clone-out-of-lock is a pointer
bump, not a 30-120 KB String copy (S10). player_manager lock/memo untouched.
Deferred (documented): S7 (drop retained player_code = a memo-state-machine
change, not a safe copy cut), S3 ($fields narrowing needs a live check).
Wrapper-crate follow-ups (out of core scope): S4 (net.rs from_utf8 fast path),
S5 (opt-level=2 for the JS-interpreter crates in rust/Cargo.toml).
156 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
parent
b5dde59464
commit
bb1038c370
4 changed files with 108 additions and 23 deletions
|
|
@ -54,6 +54,15 @@ impl Response {
|
|||
&self.response_body
|
||||
}
|
||||
|
||||
/// Consumes the response, handing back the owned body `String` with no
|
||||
/// copy. Use at the last read of a body when the rest of the `Response`
|
||||
/// is no longer needed — e.g. the ~1.5 MB player.js download, where
|
||||
/// `response_body().to_string()` would clone the whole buffer only to
|
||||
/// drop the original (S8 speed fix).
|
||||
pub fn into_body(self) -> String {
|
||||
self.response_body
|
||||
}
|
||||
|
||||
pub fn latest_url(&self) -> &str {
|
||||
&self.latest_url
|
||||
}
|
||||
|
|
|
|||
|
|
@ -136,7 +136,9 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result<St
|
|||
resp.response_code()
|
||||
)));
|
||||
}
|
||||
Ok(resp.response_body().to_string())
|
||||
// Consume the owned body — no second ~1.5 MB copy (S8). `resp` is dead
|
||||
// after the response-code check above.
|
||||
Ok(resp.into_body())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@
|
|||
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::youtube::js::extractor;
|
||||
|
|
@ -155,9 +156,13 @@ struct ManagerState {
|
|||
player_code: Option<String>,
|
||||
|
||||
signature_timestamp: Option<i32>,
|
||||
sig_snippet: Option<String>,
|
||||
nsig_name: Option<String>,
|
||||
nsig_snippet: Option<String>,
|
||||
// Snippets held as `Arc<str>` so the per-eval hand-off out of the state
|
||||
// lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only
|
||||
// the cloned payload type changes — the lock/memo/fetch_gate discipline is
|
||||
// exactly as Loop-1 left it.
|
||||
sig_snippet: Option<Arc<str>>,
|
||||
nsig_name: Option<Arc<str>>,
|
||||
nsig_snippet: Option<Arc<str>>,
|
||||
throttling_param_cache: HashMap<String, String>,
|
||||
|
||||
// Failure memos — deliberately NOT cleared by invalidate(): they must
|
||||
|
|
@ -228,7 +233,7 @@ impl PlayerManager {
|
|||
video_id,
|
||||
Derived::Sig,
|
||||
|state| state.sig_snippet.clone(),
|
||||
signature::build_deobfuscator,
|
||||
|code| signature::build_deobfuscator(code).map(Arc::<str>::from),
|
||||
|state, snippet| state.sig_snippet = Some(snippet.clone()),
|
||||
)?;
|
||||
|
||||
|
|
@ -275,7 +280,10 @@ impl PlayerManager {
|
|||
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
|
||||
_ => None,
|
||||
},
|
||||
nsig::build_deobfuscator,
|
||||
|code| {
|
||||
nsig::build_deobfuscator(code)
|
||||
.map(|(n, s)| (Arc::<str>::from(n), Arc::<str>::from(s)))
|
||||
},
|
||||
|state, (n, s)| {
|
||||
state.nsig_name = Some(n.clone());
|
||||
state.nsig_snippet = Some(s.clone());
|
||||
|
|
|
|||
|
|
@ -121,16 +121,47 @@ pub fn stream_info_with(
|
|||
);
|
||||
|
||||
let android_cpn = generate_content_playback_nonce();
|
||||
let player_response = fetch_android(
|
||||
video_id,
|
||||
&localization,
|
||||
&content_country,
|
||||
&android_cpn,
|
||||
android_token
|
||||
.as_ref()
|
||||
.map(|t| t.player_request_po_token.as_str()),
|
||||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||
)?;
|
||||
|
||||
// S2 (Option A — the conservative subset): overlap the player.js
|
||||
// signature-timestamp resolution with the primary Android reel fetch. The
|
||||
// two are fully independent — `signature_timestamp` needs only `video_id`,
|
||||
// and the WEB metadata leg that consumes the timestamp still runs AFTER the
|
||||
// decoy gate below. On a cold open (or after a player.js rotation) this
|
||||
// overlaps the ~1.5 MB base.js download with the Android round-trip instead
|
||||
// of paying them serially; warm, the timestamp is cached and the worker
|
||||
// returns near-instantly. `scope` joins both before either result is read,
|
||||
// so the borrowed locals stay valid (same shape as `channel.rs`). The
|
||||
// worker leg is best-effort: a failed extraction OR a panicked worker yields
|
||||
// 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the
|
||||
// gating primary (its playability/decoy check below is unchanged and still
|
||||
// aborts before any WEB/visionOS work), the visitorData-first ordering is
|
||||
// untouched (the player.js fetch never hits the visitor_id endpoint), and
|
||||
// player.js concurrency is already guarded by PlayerManager's fetch_gate so
|
||||
// a second concurrent open dedups the download rather than racing it.
|
||||
// Trade-off (accepted): on a COLD cache, a *failing* open can't return its
|
||||
// error until the speculative player.js leg joins (bounded by one download);
|
||||
// the win is the warm/success path. Worst case is a rare, bounded cold-fail
|
||||
// delay — never a change in what's extracted.
|
||||
let (android_result, signature_timestamp) = std::thread::scope(|s| {
|
||||
let sig_ts = s.spawn(|| {
|
||||
PlayerManager::instance()
|
||||
.signature_timestamp(video_id)
|
||||
.unwrap_or(0)
|
||||
});
|
||||
let android = fetch_android(
|
||||
video_id,
|
||||
&localization,
|
||||
&content_country,
|
||||
&android_cpn,
|
||||
android_token
|
||||
.as_ref()
|
||||
.map(|t| t.player_request_po_token.as_str()),
|
||||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||
);
|
||||
// A panicked worker joins as Err → 0, same as an extraction failure.
|
||||
(android, sig_ts.join().unwrap_or(0))
|
||||
});
|
||||
let player_response = android_result?;
|
||||
|
||||
// A primary-android rejection (playability failure or decoy) is exactly the
|
||||
// symptom a poisoned visitorData would produce on the primary reel call —
|
||||
|
|
@ -225,9 +256,9 @@ pub fn stream_info_with(
|
|||
.map(|t| t.streaming_data_po_token.clone())
|
||||
.or_else(|| options.ios_streaming_pot.clone());
|
||||
|
||||
let signature_timestamp = PlayerManager::instance()
|
||||
.signature_timestamp(video_id)
|
||||
.unwrap_or(0);
|
||||
// `signature_timestamp` was resolved concurrently with the Android fetch
|
||||
// above (S2). The WEB metadata leg is the sole consumer and runs only now,
|
||||
// after the Android decoy gate has passed.
|
||||
let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp);
|
||||
|
||||
let mut info = StreamInfo {
|
||||
|
|
@ -371,7 +402,7 @@ fn fetch_android(
|
|||
po_token: Option<&str>,
|
||||
visitor_data: Option<&str>,
|
||||
) -> Result<Value, ExtractionError> {
|
||||
let result = if po_token.is_some() {
|
||||
if po_token.is_some() {
|
||||
stream_helper::get_android_player_response(
|
||||
video_id,
|
||||
localization,
|
||||
|
|
@ -388,9 +419,23 @@ fn fetch_android(
|
|||
cpn,
|
||||
)?;
|
||||
// The reel endpoint returns the `playerResponse` nested one level.
|
||||
Ok(r.get("playerResponse").cloned().unwrap_or(r))
|
||||
};
|
||||
result
|
||||
Ok(unwrap_reel_player_response(r))
|
||||
}
|
||||
}
|
||||
|
||||
/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the
|
||||
/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead
|
||||
/// after this call, so `serde_json::Map::remove` returns the owned subtree with
|
||||
/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map /
|
||||
/// Vec / String node on every video open AND every feed-enrich (S1). Behavior
|
||||
/// is identical, including the defensive "no `playerResponse` key → use the
|
||||
/// whole response" fallback. Only the anonymous reel path calls this; the
|
||||
/// token /player path returns the response already un-nested.
|
||||
fn unwrap_reel_player_response(r: Value) -> Value {
|
||||
match r {
|
||||
Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn fetch_web_metadata(
|
||||
|
|
@ -1015,6 +1060,27 @@ mod tests {
|
|||
assert!(!is_player_response_not_valid(&resp, "MATCHING"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reel_player_response_moved_out_or_falls_back() {
|
||||
// Nested playerResponse is unwrapped (moved out, not cloned) — the
|
||||
// returned subtree is exactly the inner object.
|
||||
let r = json!({
|
||||
"playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}},
|
||||
"responseContext": {"x": 1}
|
||||
});
|
||||
let out = unwrap_reel_player_response(r);
|
||||
assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}}));
|
||||
|
||||
// No playerResponse key → the whole response is returned unchanged
|
||||
// (matches the prior `.cloned().unwrap_or(r)` fallback).
|
||||
let r = json!({"videoDetails": {"videoId": "xyz"}});
|
||||
assert_eq!(unwrap_reel_player_response(r.clone()), r);
|
||||
|
||||
// Defensive: a non-object value passes through as-is.
|
||||
let r = json!("scalar");
|
||||
assert_eq!(unwrap_reel_player_response(r.clone()), r);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cipher_string_parsed() {
|
||||
let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue