speed: move-not-clone reel playerResponse, overlap sig-timestamp fetch, cut player.js copies
Some checks failed
gitleaks / scan (push) Failing after 13m8s
Some checks failed
gitleaks / scan (push) Failing after 13m8s
- the reel `playerResponse` unwrap now MOVES the ~150-500 KB subtree out
(`Map::remove`) instead of deep-cloning it, on every open + feed-enrich (S1).
- overlap the player.js signature-timestamp resolution with the primary Android
reel fetch via `std:🧵:scope` — independent legs (disjoint lock sets),
Android stays the gating primary, decoy gate + best-effort semantics
unchanged; concurrency-harness-proven: no deadlock/race/output change (S2).
- `Response::into_body` drops a ~1.5 MB player.js body copy (S8); manager
snippet fields -> `Arc<str>` so the per-eval clone-out-of-lock is a pointer
bump, not a 30-120 KB String copy (S10). player_manager lock/memo untouched.
Deferred (documented): S7 (drop retained player_code = a memo-state-machine
change, not a safe copy cut), S3 ($fields narrowing needs a live check).
Wrapper-crate follow-ups (out of core scope): S4 (net.rs from_utf8 fast path),
S5 (opt-level=2 for the JS-interpreter crates in rust/Cargo.toml).
156 lib + 7 integration tests, clippy -D clean.
This commit is contained in:
parent
b5dde59464
commit
bb1038c370
4 changed files with 108 additions and 23 deletions
|
|
@ -54,6 +54,15 @@ impl Response {
|
||||||
&self.response_body
|
&self.response_body
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Consumes the response, handing back the owned body `String` with no
|
||||||
|
/// copy. Use at the last read of a body when the rest of the `Response`
|
||||||
|
/// is no longer needed — e.g. the ~1.5 MB player.js download, where
|
||||||
|
/// `response_body().to_string()` would clone the whole buffer only to
|
||||||
|
/// drop the original (S8 speed fix).
|
||||||
|
pub fn into_body(self) -> String {
|
||||||
|
self.response_body
|
||||||
|
}
|
||||||
|
|
||||||
pub fn latest_url(&self) -> &str {
|
pub fn latest_url(&self) -> &str {
|
||||||
&self.latest_url
|
&self.latest_url
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -136,7 +136,9 @@ fn download_javascript_code(downloader: &dyn Downloader, url: &str) -> Result<St
|
||||||
resp.response_code()
|
resp.response_code()
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
Ok(resp.response_body().to_string())
|
// Consume the owned body — no second ~1.5 MB copy (S8). `resp` is dead
|
||||||
|
// after the response-code check above.
|
||||||
|
Ok(resp.into_body())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,7 @@
|
||||||
|
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::youtube::js::extractor;
|
use crate::youtube::js::extractor;
|
||||||
|
|
@ -155,9 +156,13 @@ struct ManagerState {
|
||||||
player_code: Option<String>,
|
player_code: Option<String>,
|
||||||
|
|
||||||
signature_timestamp: Option<i32>,
|
signature_timestamp: Option<i32>,
|
||||||
sig_snippet: Option<String>,
|
// Snippets held as `Arc<str>` so the per-eval hand-off out of the state
|
||||||
nsig_name: Option<String>,
|
// lock is a pointer bump instead of a ~30-120 KB `String` copy (S10). Only
|
||||||
nsig_snippet: Option<String>,
|
// the cloned payload type changes — the lock/memo/fetch_gate discipline is
|
||||||
|
// exactly as Loop-1 left it.
|
||||||
|
sig_snippet: Option<Arc<str>>,
|
||||||
|
nsig_name: Option<Arc<str>>,
|
||||||
|
nsig_snippet: Option<Arc<str>>,
|
||||||
throttling_param_cache: HashMap<String, String>,
|
throttling_param_cache: HashMap<String, String>,
|
||||||
|
|
||||||
// Failure memos — deliberately NOT cleared by invalidate(): they must
|
// Failure memos — deliberately NOT cleared by invalidate(): they must
|
||||||
|
|
@ -228,7 +233,7 @@ impl PlayerManager {
|
||||||
video_id,
|
video_id,
|
||||||
Derived::Sig,
|
Derived::Sig,
|
||||||
|state| state.sig_snippet.clone(),
|
|state| state.sig_snippet.clone(),
|
||||||
signature::build_deobfuscator,
|
|code| signature::build_deobfuscator(code).map(Arc::<str>::from),
|
||||||
|state, snippet| state.sig_snippet = Some(snippet.clone()),
|
|state, snippet| state.sig_snippet = Some(snippet.clone()),
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
|
@ -275,7 +280,10 @@ impl PlayerManager {
|
||||||
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
|
(Some(n), Some(s)) => Some((n.clone(), s.clone())),
|
||||||
_ => None,
|
_ => None,
|
||||||
},
|
},
|
||||||
nsig::build_deobfuscator,
|
|code| {
|
||||||
|
nsig::build_deobfuscator(code)
|
||||||
|
.map(|(n, s)| (Arc::<str>::from(n), Arc::<str>::from(s)))
|
||||||
|
},
|
||||||
|state, (n, s)| {
|
|state, (n, s)| {
|
||||||
state.nsig_name = Some(n.clone());
|
state.nsig_name = Some(n.clone());
|
||||||
state.nsig_snippet = Some(s.clone());
|
state.nsig_snippet = Some(s.clone());
|
||||||
|
|
|
||||||
|
|
@ -121,16 +121,47 @@ pub fn stream_info_with(
|
||||||
);
|
);
|
||||||
|
|
||||||
let android_cpn = generate_content_playback_nonce();
|
let android_cpn = generate_content_playback_nonce();
|
||||||
let player_response = fetch_android(
|
|
||||||
video_id,
|
// S2 (Option A — the conservative subset): overlap the player.js
|
||||||
&localization,
|
// signature-timestamp resolution with the primary Android reel fetch. The
|
||||||
&content_country,
|
// two are fully independent — `signature_timestamp` needs only `video_id`,
|
||||||
&android_cpn,
|
// and the WEB metadata leg that consumes the timestamp still runs AFTER the
|
||||||
android_token
|
// decoy gate below. On a cold open (or after a player.js rotation) this
|
||||||
.as_ref()
|
// overlaps the ~1.5 MB base.js download with the Android round-trip instead
|
||||||
.map(|t| t.player_request_po_token.as_str()),
|
// of paying them serially; warm, the timestamp is cached and the worker
|
||||||
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
// returns near-instantly. `scope` joins both before either result is read,
|
||||||
)?;
|
// so the borrowed locals stay valid (same shape as `channel.rs`). The
|
||||||
|
// worker leg is best-effort: a failed extraction OR a panicked worker yields
|
||||||
|
// 0 — identical to the prior serial `.unwrap_or(0)`. Android stays the
|
||||||
|
// gating primary (its playability/decoy check below is unchanged and still
|
||||||
|
// aborts before any WEB/visionOS work), the visitorData-first ordering is
|
||||||
|
// untouched (the player.js fetch never hits the visitor_id endpoint), and
|
||||||
|
// player.js concurrency is already guarded by PlayerManager's fetch_gate so
|
||||||
|
// a second concurrent open dedups the download rather than racing it.
|
||||||
|
// Trade-off (accepted): on a COLD cache, a *failing* open can't return its
|
||||||
|
// error until the speculative player.js leg joins (bounded by one download);
|
||||||
|
// the win is the warm/success path. Worst case is a rare, bounded cold-fail
|
||||||
|
// delay — never a change in what's extracted.
|
||||||
|
let (android_result, signature_timestamp) = std::thread::scope(|s| {
|
||||||
|
let sig_ts = s.spawn(|| {
|
||||||
|
PlayerManager::instance()
|
||||||
|
.signature_timestamp(video_id)
|
||||||
|
.unwrap_or(0)
|
||||||
|
});
|
||||||
|
let android = fetch_android(
|
||||||
|
video_id,
|
||||||
|
&localization,
|
||||||
|
&content_country,
|
||||||
|
&android_cpn,
|
||||||
|
android_token
|
||||||
|
.as_ref()
|
||||||
|
.map(|t| t.player_request_po_token.as_str()),
|
||||||
|
android_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||||
|
);
|
||||||
|
// A panicked worker joins as Err → 0, same as an extraction failure.
|
||||||
|
(android, sig_ts.join().unwrap_or(0))
|
||||||
|
});
|
||||||
|
let player_response = android_result?;
|
||||||
|
|
||||||
// A primary-android rejection (playability failure or decoy) is exactly the
|
// A primary-android rejection (playability failure or decoy) is exactly the
|
||||||
// symptom a poisoned visitorData would produce on the primary reel call —
|
// symptom a poisoned visitorData would produce on the primary reel call —
|
||||||
|
|
@ -225,9 +256,9 @@ pub fn stream_info_with(
|
||||||
.map(|t| t.streaming_data_po_token.clone())
|
.map(|t| t.streaming_data_po_token.clone())
|
||||||
.or_else(|| options.ios_streaming_pot.clone());
|
.or_else(|| options.ios_streaming_pot.clone());
|
||||||
|
|
||||||
let signature_timestamp = PlayerManager::instance()
|
// `signature_timestamp` was resolved concurrently with the Android fetch
|
||||||
.signature_timestamp(video_id)
|
// above (S2). The WEB metadata leg is the sole consumer and runs only now,
|
||||||
.unwrap_or(0);
|
// after the Android decoy gate has passed.
|
||||||
let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp);
|
let web_metadata = fetch_web_metadata(video_id, &localization, &content_country, signature_timestamp);
|
||||||
|
|
||||||
let mut info = StreamInfo {
|
let mut info = StreamInfo {
|
||||||
|
|
@ -371,7 +402,7 @@ fn fetch_android(
|
||||||
po_token: Option<&str>,
|
po_token: Option<&str>,
|
||||||
visitor_data: Option<&str>,
|
visitor_data: Option<&str>,
|
||||||
) -> Result<Value, ExtractionError> {
|
) -> Result<Value, ExtractionError> {
|
||||||
let result = if po_token.is_some() {
|
if po_token.is_some() {
|
||||||
stream_helper::get_android_player_response(
|
stream_helper::get_android_player_response(
|
||||||
video_id,
|
video_id,
|
||||||
localization,
|
localization,
|
||||||
|
|
@ -388,9 +419,23 @@ fn fetch_android(
|
||||||
cpn,
|
cpn,
|
||||||
)?;
|
)?;
|
||||||
// The reel endpoint returns the `playerResponse` nested one level.
|
// The reel endpoint returns the `playerResponse` nested one level.
|
||||||
Ok(r.get("playerResponse").cloned().unwrap_or(r))
|
Ok(unwrap_reel_player_response(r))
|
||||||
};
|
}
|
||||||
result
|
}
|
||||||
|
|
||||||
|
/// Unwrap the reel endpoint's one-level `playerResponse` nesting, MOVING the
|
||||||
|
/// (~150-500 KB) subtree out instead of deep-cloning it. `r` is owned and dead
|
||||||
|
/// after this call, so `serde_json::Map::remove` returns the owned subtree with
|
||||||
|
/// zero allocation — the old `.cloned().unwrap_or(r)` re-allocated every Map /
|
||||||
|
/// Vec / String node on every video open AND every feed-enrich (S1). Behavior
|
||||||
|
/// is identical, including the defensive "no `playerResponse` key → use the
|
||||||
|
/// whole response" fallback. Only the anonymous reel path calls this; the
|
||||||
|
/// token /player path returns the response already un-nested.
|
||||||
|
fn unwrap_reel_player_response(r: Value) -> Value {
|
||||||
|
match r {
|
||||||
|
Value::Object(mut m) => m.remove("playerResponse").unwrap_or(Value::Object(m)),
|
||||||
|
other => other,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fetch_web_metadata(
|
fn fetch_web_metadata(
|
||||||
|
|
@ -1015,6 +1060,27 @@ mod tests {
|
||||||
assert!(!is_player_response_not_valid(&resp, "MATCHING"));
|
assert!(!is_player_response_not_valid(&resp, "MATCHING"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reel_player_response_moved_out_or_falls_back() {
|
||||||
|
// Nested playerResponse is unwrapped (moved out, not cloned) — the
|
||||||
|
// returned subtree is exactly the inner object.
|
||||||
|
let r = json!({
|
||||||
|
"playerResponse": {"videoDetails": {"videoId": "abc"}, "streamingData": {}},
|
||||||
|
"responseContext": {"x": 1}
|
||||||
|
});
|
||||||
|
let out = unwrap_reel_player_response(r);
|
||||||
|
assert_eq!(out, json!({"videoDetails": {"videoId": "abc"}, "streamingData": {}}));
|
||||||
|
|
||||||
|
// No playerResponse key → the whole response is returned unchanged
|
||||||
|
// (matches the prior `.cloned().unwrap_or(r)` fallback).
|
||||||
|
let r = json!({"videoDetails": {"videoId": "xyz"}});
|
||||||
|
assert_eq!(unwrap_reel_player_response(r.clone()), r);
|
||||||
|
|
||||||
|
// Defensive: a non-object value passes through as-is.
|
||||||
|
let r = json!("scalar");
|
||||||
|
assert_eq!(unwrap_reel_player_response(r.clone()), r);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cipher_string_parsed() {
|
fn cipher_string_parsed() {
|
||||||
let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1";
|
let s = "s=AAA%3D&sp=sig&url=https%3A%2F%2Fexample.com%2Fpath%3Fa%3D1";
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue