perf(extract): borrow instead of clone + parallelize channel browse
Four allocation/latency wins in the hot extraction paths. Behavior preserved — all 120 in-crate tests green, clippy clean. - stream_extractor: borrow streamingData out of the owned android + ios player responses instead of deep-cloning the largest subtree of each response. A shared Value::Null static backs the absent case. - stream_extractor: merge_formats returns borrowed &Value format objects rather than cloning each one (~20-40 per video) — they are only read (.get()) downstream. - channel: fetch the Home and Videos tabs concurrently via thread::scope so a channel open costs one round-trip of latency, not two. Home stays mandatory; the Videos tab stays best-effort (now also resilient to a panicked worker thread). - downloader: build the response body with String::from_utf8 (in-place on the valid-UTF-8 common case) instead of from_utf8_lossy, which always copies; the lossy fallback for invalid bytes is preserved.
This commit is contained in:
parent
820daec026
commit
5e0ec08928
3 changed files with 95 additions and 43 deletions
|
|
@ -117,7 +117,14 @@ impl Downloader for ReqwestDownloader {
|
||||||
"response body exceeded cap {MAX_BODY_BYTES}"
|
"response body exceeded cap {MAX_BODY_BYTES}"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let body = String::from_utf8_lossy(&buf).into_owned();
|
// Reuse the body buffer on the valid-UTF-8 path (the overwhelming
|
||||||
|
// common case): String::from_utf8 reinterprets the Vec in place,
|
||||||
|
// whereas from_utf8_lossy always allocates + copies. Fall back to
|
||||||
|
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
|
||||||
|
let body = match String::from_utf8(buf) {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||||||
|
};
|
||||||
|
|
||||||
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -94,14 +94,26 @@ pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, Extrac
|
||||||
const CHANNEL_VIDEOS_TAB_PARAMS: &str = "EgZ2aWRlb3PyBgQKAjoA";
|
const CHANNEL_VIDEOS_TAB_PARAMS: &str = "EgZ2aWRlb3PyBgQKAjoA";
|
||||||
|
|
||||||
pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionError> {
|
pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionError> {
|
||||||
// First browse — Home tab. Gives us channel header + metadata. YT
|
// The Home tab (header + metadata) and the Videos tab are two
|
||||||
// doesn't ship video items here for most channels in 2026.
|
// independent InnerTube POSTs. Run them concurrently so a channel open
|
||||||
let home_response = fetch_browse(channel_id, None)?;
|
// costs one round-trip of latency instead of two. `scope` joins both
|
||||||
|
// threads before returning, so the borrowed `channel_id` stays valid;
|
||||||
|
// the Downloader + NewPipe globals are Send+Sync and the browse path
|
||||||
|
// touches no thread-local / !Sync state.
|
||||||
|
let (home_result, videos_result) = std::thread::scope(|s| {
|
||||||
|
// Videos tab on a worker; Home tab on this thread.
|
||||||
|
let videos = s.spawn(|| fetch_browse(channel_id, Some(CHANNEL_VIDEOS_TAB_PARAMS)));
|
||||||
|
let home = fetch_browse(channel_id, None);
|
||||||
|
(home, videos.join())
|
||||||
|
});
|
||||||
|
|
||||||
|
// Home tab is mandatory — its failure fails the whole call.
|
||||||
|
let home_response = home_result?;
|
||||||
let mut info = parse_channel_browse(channel_id, &home_response);
|
let mut info = parse_channel_browse(channel_id, &home_response);
|
||||||
|
|
||||||
// Second browse — Videos tab. Best-effort: any failure here just
|
// Videos tab is best-effort: a fetch error OR a panicked worker thread
|
||||||
// leaves recent_videos empty (header still populated from first browse).
|
// just leaves recent_videos empty (header still populated above).
|
||||||
if let Ok(videos_response) = fetch_browse(channel_id, Some(CHANNEL_VIDEOS_TAB_PARAMS)) {
|
if let Ok(Ok(videos_response)) = videos_result {
|
||||||
info.recent_videos = parse_videos_tab(&videos_response);
|
info.recent_videos = parse_videos_tab(&videos_response);
|
||||||
if let Some(token) = parse_videos_continuation(&videos_response) {
|
if let Some(token) = parse_videos_continuation(&videos_response) {
|
||||||
info.videos_continuation = Some(token);
|
info.videos_continuation = Some(token);
|
||||||
|
|
|
||||||
|
|
@ -37,6 +37,12 @@ use crate::youtube::js::PlayerManager;
|
||||||
use crate::youtube::potoken::{po_token_provider, PoTokenResult};
|
use crate::youtube::potoken::{po_token_provider, PoTokenResult};
|
||||||
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
|
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
|
||||||
|
|
||||||
|
/// Shared `Value::Null` sentinel so a missing `streamingData` can be handed
|
||||||
|
/// back as a borrowed `&Value` instead of cloning the (large) present
|
||||||
|
/// subtree. An inline `&Value::Null` would be a temporary (E0716); this
|
||||||
|
/// `'static` lives for the program.
|
||||||
|
static NULL_VALUE: Value = Value::Null;
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub enum FetchPolicy {
|
pub enum FetchPolicy {
|
||||||
AnonymousAndroidReel,
|
AnonymousAndroidReel,
|
||||||
|
|
@ -102,10 +108,10 @@ pub fn stream_info_with(
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let android_streaming_data = player_response
|
// Borrow streamingData out of the (owned, function-lived) player
|
||||||
.get("streamingData")
|
// response rather than deep-cloning the largest subtree of the response.
|
||||||
.cloned()
|
let android_streaming_data: &Value =
|
||||||
.unwrap_or(Value::Null);
|
player_response.get("streamingData").unwrap_or(&NULL_VALUE);
|
||||||
|
|
||||||
// Optional iOS — best-effort.
|
// Optional iOS — best-effort.
|
||||||
let ios_token: Option<PoTokenResult> = if options.fetch_ios_client {
|
let ios_token: Option<PoTokenResult> = if options.fetch_ios_client {
|
||||||
|
|
@ -123,7 +129,9 @@ pub fn stream_info_with(
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
let (ios_streaming_data, ios_cpn) = if options.fetch_ios_client {
|
// Keep the owned iOS player response alive for the rest of the function
|
||||||
|
// so its streamingData subtree can be borrowed (not deep-cloned) below.
|
||||||
|
let (ios_response, ios_cpn): (Option<Value>, Option<String>) = if options.fetch_ios_client {
|
||||||
let ios_cpn = generate_content_playback_nonce();
|
let ios_cpn = generate_content_playback_nonce();
|
||||||
match stream_helper::get_ios_player_response(
|
match stream_helper::get_ios_player_response(
|
||||||
video_id,
|
video_id,
|
||||||
|
|
@ -133,15 +141,16 @@ pub fn stream_info_with(
|
||||||
ios_token.as_ref().map(|t| t.player_request_po_token.as_str()),
|
ios_token.as_ref().map(|t| t.player_request_po_token.as_str()),
|
||||||
ios_token.as_ref().map(|t| t.visitor_data.as_str()),
|
ios_token.as_ref().map(|t| t.visitor_data.as_str()),
|
||||||
) {
|
) {
|
||||||
Ok(r) if !is_player_response_not_valid(&r, video_id) => (
|
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(ios_cpn)),
|
||||||
r.get("streamingData").cloned().unwrap_or(Value::Null),
|
_ => (None, None),
|
||||||
Some(ios_cpn),
|
|
||||||
),
|
|
||||||
_ => (Value::Null, None),
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
(Value::Null, None)
|
(None, None)
|
||||||
};
|
};
|
||||||
|
let ios_streaming_data: &Value = ios_response
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|r| r.get("streamingData"))
|
||||||
|
.unwrap_or(&NULL_VALUE);
|
||||||
|
|
||||||
let android_streaming_pot = android_token
|
let android_streaming_pot = android_token
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|
@ -169,8 +178,8 @@ pub fn stream_info_with(
|
||||||
populate_microformat(&mut info, &web_metadata);
|
populate_microformat(&mut info, &web_metadata);
|
||||||
populate_streams(
|
populate_streams(
|
||||||
&mut info,
|
&mut info,
|
||||||
&android_streaming_data,
|
android_streaming_data,
|
||||||
&ios_streaming_data,
|
ios_streaming_data,
|
||||||
video_id,
|
video_id,
|
||||||
&android_cpn,
|
&android_cpn,
|
||||||
ios_cpn.as_deref(),
|
ios_cpn.as_deref(),
|
||||||
|
|
@ -179,8 +188,8 @@ pub fn stream_info_with(
|
||||||
)?;
|
)?;
|
||||||
populate_manifests(
|
populate_manifests(
|
||||||
&mut info,
|
&mut info,
|
||||||
&android_streaming_data,
|
android_streaming_data,
|
||||||
&ios_streaming_data,
|
ios_streaming_data,
|
||||||
android_streaming_pot.as_deref(),
|
android_streaming_pot.as_deref(),
|
||||||
ios_streaming_pot.as_deref(),
|
ios_streaming_pot.as_deref(),
|
||||||
);
|
);
|
||||||
|
|
@ -390,6 +399,36 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Merge the Android + iOS format objects under one streamingData array
|
||||||
|
/// key (`formats` / `adaptiveFormats`), each tagged with its client + cpn +
|
||||||
|
/// pot. Returns borrowed `&Value`s into the player responses: the format
|
||||||
|
/// objects are only ever read (`.get(...)`) downstream, so cloning the whole
|
||||||
|
/// array (~20-40 objects per video) was pure waste.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn merge_formats<'a>(
|
||||||
|
android: &'a Value,
|
||||||
|
ios: &'a Value,
|
||||||
|
fmt_array_key: &str,
|
||||||
|
android_cpn: &'a str,
|
||||||
|
ios_cpn: Option<&'a str>,
|
||||||
|
android_pot: Option<&'a str>,
|
||||||
|
ios_pot: Option<&'a str>,
|
||||||
|
) -> Vec<(&'a Value, &'static str, &'a str, Option<&'a str>)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) {
|
||||||
|
for f in arr {
|
||||||
|
out.push((f, "ANDROID", android_cpn, android_pot));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) {
|
||||||
|
let cpn = ios_cpn.unwrap_or("");
|
||||||
|
for f in arr {
|
||||||
|
out.push((f, "IOS", cpn, ios_pot));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
fn populate_streams(
|
fn populate_streams(
|
||||||
info: &mut StreamInfo,
|
info: &mut StreamInfo,
|
||||||
|
|
@ -401,41 +440,35 @@ fn populate_streams(
|
||||||
android_pot: Option<&str>,
|
android_pot: Option<&str>,
|
||||||
ios_pot: Option<&str>,
|
ios_pot: Option<&str>,
|
||||||
) -> Result<(), ExtractionError> {
|
) -> Result<(), ExtractionError> {
|
||||||
let merge = |fmt_array_key: &str| -> Vec<(Value, &'static str, &str, Option<&str>)> {
|
|
||||||
let mut out = Vec::new();
|
|
||||||
if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) {
|
|
||||||
for f in arr {
|
|
||||||
out.push((f.clone(), "ANDROID", android_cpn, android_pot));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) {
|
|
||||||
for f in arr {
|
|
||||||
let cpn = ios_cpn.unwrap_or("");
|
|
||||||
out.push((f.clone(), "IOS", cpn, ios_pot));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out
|
|
||||||
};
|
|
||||||
|
|
||||||
// Progressive: streamingData.formats[]
|
// Progressive: streamingData.formats[]
|
||||||
for (fmt, _client, cpn, pot) in merge("formats") {
|
for (fmt, _client, cpn, pot) in
|
||||||
if let Some(stream) = build_video_progressive(&fmt, video_id, cpn, pot)? {
|
merge_formats(android, ios, "formats", android_cpn, ios_cpn, android_pot, ios_pot)
|
||||||
|
{
|
||||||
|
if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? {
|
||||||
push_video_dedup(&mut info.video_streams, stream);
|
push_video_dedup(&mut info.video_streams, stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Adaptive: streamingData.adaptiveFormats[]
|
// Adaptive: streamingData.adaptiveFormats[]
|
||||||
for (fmt, _client, cpn, pot) in merge("adaptiveFormats") {
|
for (fmt, _client, cpn, pot) in merge_formats(
|
||||||
|
android,
|
||||||
|
ios,
|
||||||
|
"adaptiveFormats",
|
||||||
|
android_cpn,
|
||||||
|
ios_cpn,
|
||||||
|
android_pot,
|
||||||
|
ios_pot,
|
||||||
|
) {
|
||||||
let mime = fmt
|
let mime = fmt
|
||||||
.get("mimeType")
|
.get("mimeType")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("");
|
.unwrap_or("");
|
||||||
if mime.starts_with("audio/") {
|
if mime.starts_with("audio/") {
|
||||||
if let Some(audio) = build_audio(&fmt, video_id, cpn, pot)? {
|
if let Some(audio) = build_audio(fmt, video_id, cpn, pot)? {
|
||||||
push_audio_dedup(&mut info.audio_streams, audio);
|
push_audio_dedup(&mut info.audio_streams, audio);
|
||||||
}
|
}
|
||||||
} else if mime.starts_with("video/") {
|
} else if mime.starts_with("video/") {
|
||||||
if let Some(video) = build_video_only(&fmt, video_id, cpn, pot)? {
|
if let Some(video) = build_video_only(fmt, video_id, cpn, pot)? {
|
||||||
push_video_dedup(&mut info.video_only_streams, video);
|
push_video_dedup(&mut info.video_only_streams, video);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue