Sulkta-OSS/strawcore
3
0
Fork 0
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

perf(extract): borrow instead of clone + parallelize channel browse

Browse source 
Four allocation/latency wins in the hot extraction paths. Behavior
preserved — all 120 in-crate tests green, clippy clean.

- stream_extractor: borrow streamingData out of the owned android + ios
  player responses instead of deep-cloning the largest subtree of each
  response. A shared Value::Null static backs the absent case.
- stream_extractor: merge_formats returns borrowed &Value format objects
  rather than cloning each one (~20-40 per video) — they are only read
  (.get()) downstream.
- channel: fetch the Home and Videos tabs concurrently via thread::scope
  so a channel open costs one round-trip of latency, not two. Home stays
  mandatory; the Videos tab stays best-effort (now also resilient to a
  panicked worker thread).
- downloader: build the response body with String::from_utf8 (in-place on
  the valid-UTF-8 common case) instead of from_utf8_lossy, which always
  copies; the lossy fallback for invalid bytes is preserved.
This commit is contained in:
Sulkta 2026-06-21 05:40:11 -07:00
parent 820daec026
commit 5e0ec08928
3 changed files with 95 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/downloader/default_impl.rs
View file

@ -117,7 +117,14 @@ impl Downloader for ReqwestDownloader {
"response body exceeded cap {MAX_BODY_BYTES}"
)));
}
let body = String::from_utf8_lossy(&buf).into_owned();
// Reuse the body buffer on the valid-UTF-8 path (the overwhelming
// common case): String::from_utf8 reinterprets the Vec in place,
// whereas from_utf8_lossy always allocates + copies. Fall back to
// lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
let body = match String::from_utf8(buf) {
Ok(s) => s,
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
};
Ok(Response::new(code, message, headers, body, url_after_redirects))
}

24
src/youtube/channel.rs
View file

@ -94,14 +94,26 @@ pub fn resolve_handle_to_channel_id(url_fragment: &str) -> Result<String, Extrac
const CHANNEL_VIDEOS_TAB_PARAMS: &str = "EgZ2aWRlb3PyBgQKAjoA";
pub fn fetch_channel_browse(channel_id: &str) -> Result<ChannelInfo, ExtractionError> {
// First browse — Home tab. Gives us channel header + metadata. YT
// doesn't ship video items here for most channels in 2026.
let home_response = fetch_browse(channel_id, None)?;
// The Home tab (header + metadata) and the Videos tab are two
// independent InnerTube POSTs. Run them concurrently so a channel open
// costs one round-trip of latency instead of two. `scope` joins both
// threads before returning, so the borrowed `channel_id` stays valid;
// the Downloader + NewPipe globals are Send+Sync and the browse path
// touches no thread-local / !Sync state.
let (home_result, videos_result) = std::thread::scope(|s| {
// Videos tab on a worker; Home tab on this thread.
let videos = s.spawn(|| fetch_browse(channel_id, Some(CHANNEL_VIDEOS_TAB_PARAMS)));
let home = fetch_browse(channel_id, None);
(home, videos.join())
});
// Home tab is mandatory — its failure fails the whole call.
let home_response = home_result?;
let mut info = parse_channel_browse(channel_id, &home_response);
// Second browse — Videos tab. Best-effort: any failure here just
// leaves recent_videos empty (header still populated from first browse).
if let Ok(videos_response) = fetch_browse(channel_id, Some(CHANNEL_VIDEOS_TAB_PARAMS)) {
// Videos tab is best-effort: a fetch error OR a panicked worker thread
// just leaves recent_videos empty (header still populated above).
if let Ok(Ok(videos_response)) = videos_result {
info.recent_videos = parse_videos_tab(&videos_response);
if let Some(token) = parse_videos_continuation(&videos_response) {
info.videos_continuation = Some(token);

105
src/youtube/stream_extractor.rs
View file

@ -37,6 +37,12 @@ use crate::youtube::js::PlayerManager;
use crate::youtube::potoken::{po_token_provider, PoTokenResult};
use crate::youtube::stream_helper::{self, generate_content_playback_nonce};
/// Shared `Value::Null` sentinel so a missing `streamingData` can be handed
/// back as a borrowed `&Value` instead of cloning the (large) present
/// subtree. An inline `&Value::Null` would be a temporary (E0716); this
/// `'static` lives for the program.
static NULL_VALUE: Value = Value::Null;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FetchPolicy {
AnonymousAndroidReel,
@ -102,10 +108,10 @@ pub fn stream_info_with(
));
}
let android_streaming_data = player_response
.get("streamingData")
.cloned()
.unwrap_or(Value::Null);
// Borrow streamingData out of the (owned, function-lived) player
// response rather than deep-cloning the largest subtree of the response.
let android_streaming_data: &Value =
player_response.get("streamingData").unwrap_or(&NULL_VALUE);
// Optional iOS — best-effort.
let ios_token: Option<PoTokenResult> = if options.fetch_ios_client {
@ -123,7 +129,9 @@ pub fn stream_info_with(
None
};
let (ios_streaming_data, ios_cpn) = if options.fetch_ios_client {
// Keep the owned iOS player response alive for the rest of the function
// so its streamingData subtree can be borrowed (not deep-cloned) below.
let (ios_response, ios_cpn): (Option<Value>, Option<String>) = if options.fetch_ios_client {
let ios_cpn = generate_content_playback_nonce();
match stream_helper::get_ios_player_response(
video_id,
@ -133,15 +141,16 @@ pub fn stream_info_with(
ios_token.as_ref().map(|t| t.player_request_po_token.as_str()),
ios_token.as_ref().map(|t| t.visitor_data.as_str()),
) {
Ok(r) if !is_player_response_not_valid(&r, video_id) => (
r.get("streamingData").cloned().unwrap_or(Value::Null),
Some(ios_cpn),
),
_ => (Value::Null, None),
Ok(r) if !is_player_response_not_valid(&r, video_id) => (Some(r), Some(ios_cpn)),
_ => (None, None),
}
} else {
(Value::Null, None)
(None, None)
};
let ios_streaming_data: &Value = ios_response
.as_ref()
.and_then(|r| r.get("streamingData"))
.unwrap_or(&NULL_VALUE);
let android_streaming_pot = android_token
.as_ref()
@ -169,8 +178,8 @@ pub fn stream_info_with(
populate_microformat(&mut info, &web_metadata);
populate_streams(
&mut info,
&android_streaming_data,
&ios_streaming_data,
android_streaming_data,
ios_streaming_data,
video_id,
&android_cpn,
ios_cpn.as_deref(),
@ -179,8 +188,8 @@ pub fn stream_info_with(
)?;
populate_manifests(
&mut info,
&android_streaming_data,
&ios_streaming_data,
android_streaming_data,
ios_streaming_data,
android_streaming_pot.as_deref(),
ios_streaming_pot.as_deref(),
);
@ -390,6 +399,36 @@ fn populate_microformat(info: &mut StreamInfo, web_metadata: &Value) {
}
}
/// Merge the Android + iOS format objects under one streamingData array
/// key (`formats` / `adaptiveFormats`), each tagged with its client + cpn +
/// pot. Returns borrowed `&Value`s into the player responses: the format
/// objects are only ever read (`.get(...)`) downstream, so cloning the whole
/// array (~20-40 objects per video) was pure waste.
#[allow(clippy::too_many_arguments)]
fn merge_formats<'a>(
android: &'a Value,
ios: &'a Value,
fmt_array_key: &str,
android_cpn: &'a str,
ios_cpn: Option<&'a str>,
android_pot: Option<&'a str>,
ios_pot: Option<&'a str>,
) -> Vec<(&'a Value, &'static str, &'a str, Option<&'a str>)> {
let mut out = Vec::new();
if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) {
for f in arr {
out.push((f, "ANDROID", android_cpn, android_pot));
}
}
if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) {
let cpn = ios_cpn.unwrap_or("");
for f in arr {
out.push((f, "IOS", cpn, ios_pot));
}
}
out
}
#[allow(clippy::too_many_arguments)]
fn populate_streams(
info: &mut StreamInfo,
@ -401,41 +440,35 @@ fn populate_streams(
android_pot: Option<&str>,
ios_pot: Option<&str>,
) -> Result<(), ExtractionError> {
let merge = |fmt_array_key: &str| -> Vec<(Value, &'static str, &str, Option<&str>)> {
let mut out = Vec::new();
if let Some(arr) = android.get(fmt_array_key).and_then(|v| v.as_array()) {
for f in arr {
out.push((f.clone(), "ANDROID", android_cpn, android_pot));
}
}
if let Some(arr) = ios.get(fmt_array_key).and_then(|v| v.as_array()) {
for f in arr {
let cpn = ios_cpn.unwrap_or("");
out.push((f.clone(), "IOS", cpn, ios_pot));
}
}
out
};
// Progressive: streamingData.formats[]
for (fmt, _client, cpn, pot) in merge("formats") {
if let Some(stream) = build_video_progressive(&fmt, video_id, cpn, pot)? {
for (fmt, _client, cpn, pot) in
merge_formats(android, ios, "formats", android_cpn, ios_cpn, android_pot, ios_pot)
{
if let Some(stream) = build_video_progressive(fmt, video_id, cpn, pot)? {
push_video_dedup(&mut info.video_streams, stream);
}
}
// Adaptive: streamingData.adaptiveFormats[]
for (fmt, _client, cpn, pot) in merge("adaptiveFormats") {
for (fmt, _client, cpn, pot) in merge_formats(
android,
ios,
"adaptiveFormats",
android_cpn,
ios_cpn,
android_pot,
ios_pot,
) {
let mime = fmt
.get("mimeType")
.and_then(|v| v.as_str())
.unwrap_or("");
if mime.starts_with("audio/") {
if let Some(audio) = build_audio(&fmt, video_id, cpn, pot)? {
if let Some(audio) = build_audio(fmt, video_id, cpn, pot)? {
push_audio_dedup(&mut info.audio_streams, audio);
}
} else if mime.starts_with("video/") {
if let Some(video) = build_video_only(&fmt, video_id, cpn, pot)? {
if let Some(video) = build_video_only(fmt, video_id, cpn, pot)? {
push_video_dedup(&mut info.video_only_streams, video);
}
}