Cleanup: drop playlist + suggestion + dead client constants + suppress_unused stubs
Round-2 cruft audit punch list — mechanical deletes, no behavior change.
Whole modules deleted (no wrapper consumer):
* youtube/playlist_extractor.rs (297 LOC) — full playlist extraction
* youtube/linkhandler/playlist.rs (81 LOC) — playlist URL parser
* youtube/suggestion_extractor.rs (91 LOC) — search-as-you-type
* tests/stream_phase4_offline.rs (186 LOC) — tautological test
Dead pub fns + enum variants + constants:
* WEB_REMIX_* constants (3) + WEB_MUSIC_ANALYTICS_* constants (3)
* InnertubeClientRequestInfo::of_web_music_analytics_charts_client
factory + its charts_client_omits_platform_and_screen test
* SearchFilter::Music{Songs,Videos,Albums,Playlists,Artists} variants
(5 of 9 cases) + uses_music_endpoint helper + the search_extractor
'music search not implemented' reject branch
* Two #[allow(dead_code)] _suppress_unused stub fns and the imports
they were keeping alive (std::sync::Arc in js/extractor.rs,
NetworkError in stream_extractor.rs)
Renamed:
* search_extractor::test_helpers -> renderer_helpers. Mis-named:
it's production code called from channel.rs, not a test fixture.
potoken/ kept and documented as the designed Phase-5 extension point
for YouTube bot-detection — wrapper's Android side hasn't registered
a real provider yet, but the trait + global slot stay so when YT
forces po_token universally the integration is one Kotlin patch away,
not a Rust-side rewrite.
~580 LOC removed from production. Wrapper does not need to change.
This commit is contained in:
parent
bfd06d1ef3
commit
d4000a9f9a
13 changed files with 27 additions and 760 deletions
|
|
@ -255,7 +255,7 @@ fn parse_videos_tab(body: &Value) -> Vec<StreamInfoItem> {
|
|||
continue;
|
||||
};
|
||||
if let Some(vr) = content.get("videoRenderer") {
|
||||
if let Some(item) = crate::youtube::search_extractor::test_helpers::video_renderer_to_item(vr) {
|
||||
if let Some(item) = crate::youtube::search_extractor::renderer_helpers::video_renderer_to_item(vr) {
|
||||
out.push(item);
|
||||
}
|
||||
} else if let Some(lvm) = content.get("lockupViewModel") {
|
||||
|
|
|
|||
|
|
@ -73,23 +73,6 @@ impl InnertubeClientRequestInfo {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn of_web_music_analytics_charts_client() -> Self {
|
||||
// NPE deliberately omits clientScreen + platform for charts.
|
||||
Self {
|
||||
client_info: ClientInfo {
|
||||
client_name: WEB_MUSIC_ANALYTICS_CLIENT_NAME.into(),
|
||||
client_version: WEB_MUSIC_ANALYTICS_CLIENT_VERSION.into(),
|
||||
client_id: WEB_MUSIC_ANALYTICS_CLIENT_ID.into(),
|
||||
client_screen: None,
|
||||
visitor_data: None,
|
||||
},
|
||||
device_info: DeviceInfo {
|
||||
android_sdk_version: -1,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn of_android_client() -> Self {
|
||||
Self {
|
||||
client_info: ClientInfo {
|
||||
|
|
@ -292,21 +275,6 @@ mod tests {
|
|||
assert!(client.get("androidSdkVersion").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn charts_client_omits_platform_and_screen() {
|
||||
let info = InnertubeClientRequestInfo::of_web_music_analytics_charts_client();
|
||||
let env = build_envelope(
|
||||
&info,
|
||||
&Localization::default(),
|
||||
&ContentCountry::default(),
|
||||
None,
|
||||
);
|
||||
let client = &env["context"]["client"];
|
||||
assert_eq!(client["clientName"], "WEB_MUSIC_ANALYTICS");
|
||||
assert!(client.get("clientScreen").is_none());
|
||||
assert!(client.get("platform").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn embed_url_lands_in_third_party_block() {
|
||||
let info = InnertubeClientRequestInfo::of_web_embedded_player_client();
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
// ClientsConstants — mirrors NPE services/youtube/ClientsConstants.java.
|
||||
//
|
||||
// Six live InnerTube clients: WEB, WEB_EMBEDDED_PLAYER, WEB_MUSIC_ANALYTICS,
|
||||
// WEB_REMIX, ANDROID, IOS. NPE's tree also mentions TVHTML5 + MWEB in
|
||||
// comments — not actively used; skipped per audit Track A §1.3.
|
||||
// InnerTube client constants. Active clients: WEB, WEB_EMBEDDED_PLAYER,
|
||||
// ANDROID, IOS. WEB_REMIX (music) and WEB_MUSIC_ANALYTICS were ported
|
||||
// but never wired through — dropped 2026-05-26 along with the music
|
||||
// SearchFilter variants and the of_web_music_analytics_charts_client
|
||||
// factory.
|
||||
|
||||
pub const DESKTOP_CLIENT_PLATFORM: &str = "DESKTOP";
|
||||
pub const MOBILE_CLIENT_PLATFORM: &str = "MOBILE";
|
||||
|
|
@ -13,18 +13,10 @@ pub const WEB_CLIENT_ID: &str = "1";
|
|||
pub const WEB_CLIENT_NAME: &str = "WEB";
|
||||
pub const WEB_HARDCODED_CLIENT_VERSION: &str = "2.20260120.01.00";
|
||||
|
||||
pub const WEB_REMIX_CLIENT_ID: &str = "67";
|
||||
pub const WEB_REMIX_CLIENT_NAME: &str = "WEB_REMIX";
|
||||
pub const WEB_REMIX_HARDCODED_CLIENT_VERSION: &str = "1.20260121.03.00";
|
||||
|
||||
pub const WEB_EMBEDDED_CLIENT_ID: &str = "56";
|
||||
pub const WEB_EMBEDDED_CLIENT_NAME: &str = "WEB_EMBEDDED_PLAYER";
|
||||
pub const WEB_EMBEDDED_CLIENT_VERSION: &str = "1.20260122.01.00";
|
||||
|
||||
pub const WEB_MUSIC_ANALYTICS_CLIENT_ID: &str = "31";
|
||||
pub const WEB_MUSIC_ANALYTICS_CLIENT_NAME: &str = "WEB_MUSIC_ANALYTICS";
|
||||
pub const WEB_MUSIC_ANALYTICS_CLIENT_VERSION: &str = "2.0";
|
||||
|
||||
// iPhone 15 Pro Max — chosen explicitly for 60fps tags per ItagItem.java:26
|
||||
// note and the gist referenced in NPE source.
|
||||
pub const IOS_CLIENT_ID: &str = "5";
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@
|
|||
// Jsoup's attr-setter doesn't filter. Our walk does the same — iterate
|
||||
// every script tag, return first whose `src` contains `base.js`.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
|
||||
|
|
@ -137,9 +135,6 @@ pub fn extract_player_hash(url: &str) -> Option<String> {
|
|||
RE.captures(url).and_then(|c| c.get(1)).map(|m| m.as_str().to_string())
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn _suppress_unused_arc_import(_: Arc<dyn Downloader>) {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
// mirror list in NPE is dropped — Sulkta isn't an Invidious mirror.
|
||||
|
||||
pub mod channel;
|
||||
pub mod playlist;
|
||||
pub mod search;
|
||||
pub mod stream;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
// YoutubePlaylistLinkHandlerFactory — accepts:
|
||||
// * https://www.youtube.com/playlist?list=<PLid>
|
||||
// * https://www.youtube.com/watch?v=...&list=<PLid>
|
||||
// * https://music.youtube.com/playlist?list=<PLid>
|
||||
//
|
||||
// YT playlist IDs prefix:
|
||||
// * PL user-curated playlists
|
||||
// * RD mix / radio
|
||||
// * OLAK5uy_ album / single
|
||||
// * LL liked-videos (private — won't extract anonymously)
|
||||
// * WL watch-later (private)
|
||||
// * UU uploads (auto-generated per channel)
|
||||
|
||||
use url::Url;
|
||||
|
||||
use crate::youtube::linkhandler::{host_is_youtube, LinkError};
|
||||
|
||||
pub fn extract_playlist_id(url_str: &str) -> Result<String, LinkError> {
|
||||
let url = Url::parse(url_str)
|
||||
.map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?;
|
||||
let host = url
|
||||
.host_str()
|
||||
.ok_or_else(|| LinkError::InvalidUrl("no host".into()))?;
|
||||
if !host_is_youtube(host) {
|
||||
return Err(LinkError::UnsupportedHost(host.into()));
|
||||
}
|
||||
url.query_pairs()
|
||||
.find(|(k, _)| k == "list")
|
||||
.map(|(_, v)| v.into_owned())
|
||||
.filter(|s| !s.is_empty())
|
||||
.ok_or_else(|| LinkError::MissingId(url_str.into()))
|
||||
}
|
||||
|
||||
pub fn playlist_url(playlist_id: &str) -> String {
|
||||
format!("https://www.youtube.com/playlist?list={playlist_id}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn standalone_playlist() {
|
||||
let id = extract_playlist_id(
|
||||
"https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(id, "PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watch_with_list() {
|
||||
let id = extract_playlist_id(
|
||||
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLxxx",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(id, "PLxxx");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn music_subdomain() {
|
||||
let id = extract_playlist_id(
|
||||
"https://music.youtube.com/playlist?list=OLAK5uy_kFooBar",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(id, "OLAK5uy_kFooBar");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_no_list_param() {
|
||||
let err = extract_playlist_id("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, LinkError::MissingId(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_youtube_host() {
|
||||
let err = extract_playlist_id("https://invidious.io/playlist?list=PLxxx").unwrap_err();
|
||||
assert!(matches!(err, LinkError::UnsupportedHost(_)));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +1,11 @@
|
|||
// YoutubeSearchQueryHandlerFactory + search filters. Mirrors NPE
|
||||
// YoutubeSearchQueryHandlerFactory.java + the filter params in
|
||||
// YoutubeSearchExtractor.java.
|
||||
// YoutubeSearchQueryHandlerFactory + search filters. Filter params are
|
||||
// opaque base64 protobufs — NPE doesn't decode them, just sends the
|
||||
// magic strings. We mirror that. See audit Track D §3.
|
||||
//
|
||||
// Filter params are opaque base64 protobufs — NPE doesn't decode them,
|
||||
// just sends the magic strings. We mirror that. See audit Track D §3.
|
||||
// Music* filter variants were ported from NPE but never wired through
|
||||
// (search_extractor rejected anything with `uses_music_endpoint()`).
|
||||
// Dropped 2026-05-26 along with WEB_REMIX_* constants + the music
|
||||
// charts client factory.
|
||||
|
||||
use url::form_urlencoded;
|
||||
|
||||
|
|
@ -17,16 +19,6 @@ pub enum SearchFilter {
|
|||
Channels,
|
||||
/// Playlists only.
|
||||
Playlists,
|
||||
/// "Music songs" filter — uses the WEB_REMIX path on music.youtube.com.
|
||||
MusicSongs,
|
||||
/// "Music videos" filter — also WEB_REMIX.
|
||||
MusicVideos,
|
||||
/// "Music albums" filter.
|
||||
MusicAlbums,
|
||||
/// "Music playlists" filter.
|
||||
MusicPlaylists,
|
||||
/// "Music artists" filter.
|
||||
MusicArtists,
|
||||
}
|
||||
|
||||
impl SearchFilter {
|
||||
|
|
@ -38,24 +30,8 @@ impl SearchFilter {
|
|||
SearchFilter::Videos => Some("EgIQAfABAQ%3D%3D"),
|
||||
SearchFilter::Channels => Some("EgIQAvABAQ%3D%3D"),
|
||||
SearchFilter::Playlists => Some("EgIQA_ABAQ%3D%3D"),
|
||||
SearchFilter::MusicSongs => Some("EgWKAQIIAWoMEA4QChADEAQQCRAF"),
|
||||
SearchFilter::MusicVideos => Some("EgWKAQIQAWoMEA4QChADEAQQCRAF"),
|
||||
SearchFilter::MusicAlbums => Some("EgWKAQIYAWoMEA4QChADEAQQCRAF"),
|
||||
SearchFilter::MusicPlaylists => Some("EgeKAQQoAEABagwQDhAKEAMQBBAJEAU%3D"),
|
||||
SearchFilter::MusicArtists => Some("EgWKAQIgAWoMEA4QChADEAQQCRAF"),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn uses_music_endpoint(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
SearchFilter::MusicSongs
|
||||
| SearchFilter::MusicVideos
|
||||
| SearchFilter::MusicAlbums
|
||||
| SearchFilter::MusicPlaylists
|
||||
| SearchFilter::MusicArtists
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_url(query: &str) -> String {
|
||||
|
|
@ -81,12 +57,6 @@ mod tests {
|
|||
assert!(SearchFilter::Playlists.params().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn music_filters_route_to_music_endpoint() {
|
||||
assert!(SearchFilter::MusicSongs.uses_music_endpoint());
|
||||
assert!(!SearchFilter::Videos.uses_music_endpoint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_url_encodes_query() {
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
// YouTube service tree. Phase 2 landed the JS deobfuscator. Phase 3 adds
|
||||
// the InnerTube client matrix, request envelope, parsing helpers, and the
|
||||
// itag table. Phase 4+ will add the stream extractor, search, channel,
|
||||
// playlist, kiosks.
|
||||
// YouTube service tree: JS deobfuscator, the InnerTube client matrix +
|
||||
// request envelope, parsing helpers, itag table, and the stream / search
|
||||
// / channel extractors.
|
||||
|
||||
pub mod channel;
|
||||
pub mod client_request;
|
||||
|
|
@ -10,10 +9,14 @@ pub mod itag;
|
|||
pub mod js;
|
||||
pub mod linkhandler;
|
||||
pub mod parsing;
|
||||
pub mod playlist_extractor;
|
||||
/// PoToken bot-detection hook. `stream_extractor` reads the global
|
||||
/// provider via `po_token_provider()`; the Android side hasn't
|
||||
/// registered a real provider yet, so streams currently flow without
|
||||
/// a po_token (works until YT requires one universally). This module
|
||||
/// is the designed extension point — do not delete just because the
|
||||
/// provider is currently no-op.
|
||||
pub mod potoken;
|
||||
pub mod search_extractor;
|
||||
pub mod stream_extractor;
|
||||
pub mod stream_helper;
|
||||
pub mod suggestion_extractor;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,297 +0,0 @@
|
|||
// YoutubePlaylistExtractor — mirrors NPE
|
||||
// services/youtube/extractors/YoutubePlaylistExtractor.java.
|
||||
//
|
||||
// 2-POST pattern (audit Track D §7):
|
||||
// 1. browseId="VL<playlistId>" → playlist metadata + first batch
|
||||
// 2. continuation token → subsequent batches
|
||||
//
|
||||
// Body shape per call: build_desktop_envelope + add browseId (or
|
||||
// continuation). Response walked to playlistVideoListRenderer.contents[]
|
||||
// .playlistVideoRenderer.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::downloader::request::Request;
|
||||
use crate::exceptions::{ExtractionError, NetworkError, ParsingError};
|
||||
use crate::image::ImageSet;
|
||||
use crate::newpipe::NewPipe;
|
||||
use crate::stream::StreamInfoItem;
|
||||
use crate::youtube::client_request::build_desktop_envelope;
|
||||
use crate::youtube::constants::*;
|
||||
use crate::youtube::parsing::{web_client_version, youtube_post_headers};
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct PlaylistInfo {
|
||||
pub playlist_id: String,
|
||||
pub url: String,
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub uploader_name: String,
|
||||
pub uploader_url: String,
|
||||
pub uploader_id: String,
|
||||
pub thumbnails: ImageSet,
|
||||
pub video_count: i64,
|
||||
pub videos: Vec<StreamInfoItem>,
|
||||
pub continuation_token: Option<String>,
|
||||
}
|
||||
|
||||
pub fn playlist_info(playlist_id: &str) -> Result<PlaylistInfo, ExtractionError> {
|
||||
let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?;
|
||||
let localization = NewPipe::preferred_localization();
|
||||
let content_country = NewPipe::preferred_content_country();
|
||||
|
||||
let mut envelope =
|
||||
build_desktop_envelope(&localization, &content_country, &web_client_version());
|
||||
if let Value::Object(ref mut map) = envelope {
|
||||
map.insert(
|
||||
"browseId".into(),
|
||||
Value::String(format!("VL{playlist_id}")),
|
||||
);
|
||||
}
|
||||
let url = format!("{YOUTUBEI_V1_URL}browse{DISABLE_PRETTY_PRINT_PARAM}");
|
||||
let body = serde_json::to_vec(&envelope).map_err(|e| {
|
||||
ExtractionError::Parsing(ParsingError::Invalid(format!("serialize: {e}")))
|
||||
})?;
|
||||
let mut builder = Request::post(&url, body);
|
||||
for (k, v) in youtube_post_headers() {
|
||||
builder = builder.add_header(&k, &v);
|
||||
}
|
||||
let resp = downloader.execute(builder.build())?;
|
||||
if resp.response_code() != 200 {
|
||||
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
||||
"browse HTTP {}",
|
||||
resp.response_code()
|
||||
))));
|
||||
}
|
||||
let parsed: Value = serde_json::from_str(resp.response_body())
|
||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||
Ok(parse_playlist_browse(playlist_id, &parsed))
|
||||
}
|
||||
|
||||
pub fn parse_playlist_browse(playlist_id: &str, body: &Value) -> PlaylistInfo {
|
||||
let mut info = PlaylistInfo {
|
||||
playlist_id: playlist_id.into(),
|
||||
url: format!("https://www.youtube.com/playlist?list={playlist_id}"),
|
||||
..PlaylistInfo::default()
|
||||
};
|
||||
|
||||
// metadata.playlistMetadataRenderer.title / description
|
||||
if let Some(meta) = body
|
||||
.get("metadata")
|
||||
.and_then(|m| m.get("playlistMetadataRenderer"))
|
||||
{
|
||||
if let Some(s) = meta.get("title").and_then(|v| v.as_str()) {
|
||||
info.name = s.into();
|
||||
}
|
||||
if let Some(s) = meta.get("description").and_then(|v| v.as_str()) {
|
||||
info.description = s.into();
|
||||
}
|
||||
}
|
||||
|
||||
// sidebar.playlistSidebarRenderer.items[].playlistSidebarPrimaryInfoRenderer
|
||||
// + playlistSidebarSecondaryInfoRenderer
|
||||
if let Some(items) = body
|
||||
.get("sidebar")
|
||||
.and_then(|s| s.get("playlistSidebarRenderer"))
|
||||
.and_then(|s| s.get("items"))
|
||||
.and_then(|i| i.as_array())
|
||||
{
|
||||
for item in items {
|
||||
if let Some(primary) = item.get("playlistSidebarPrimaryInfoRenderer") {
|
||||
if info.name.is_empty() {
|
||||
if let Some(s) = primary
|
||||
.get("title")
|
||||
.and_then(|t| t.get("runs"))
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|a| a.first())
|
||||
.and_then(|r| r.get("text"))
|
||||
.and_then(|t| t.as_str())
|
||||
{
|
||||
info.name = s.into();
|
||||
}
|
||||
}
|
||||
// stats[1] (video count) — "1,234 videos"
|
||||
if let Some(stats) = primary.get("stats").and_then(|s| s.as_array()) {
|
||||
if let Some(count_text) = stats
|
||||
.get(0)
|
||||
.and_then(|s| s.get("runs"))
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|a| a.first())
|
||||
.and_then(|r| r.get("text"))
|
||||
.and_then(|t| t.as_str())
|
||||
{
|
||||
info.video_count = count_text
|
||||
.replace(',', "")
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(secondary) = item.get("playlistSidebarSecondaryInfoRenderer") {
|
||||
if let Some(owner) = secondary.get("videoOwner").and_then(|o| {
|
||||
o.get("videoOwnerRenderer")
|
||||
}) {
|
||||
if let Some(s) = owner
|
||||
.get("title")
|
||||
.and_then(|t| t.get("runs"))
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|a| a.first())
|
||||
{
|
||||
if let Some(name) = s.get("text").and_then(|t| t.as_str()) {
|
||||
info.uploader_name = name.into();
|
||||
}
|
||||
if let Some(endpoint) = s.get("navigationEndpoint") {
|
||||
if let Some(browse_id) = endpoint
|
||||
.get("browseEndpoint")
|
||||
.and_then(|b| b.get("browseId"))
|
||||
.and_then(|i| i.as_str())
|
||||
{
|
||||
info.uploader_id = browse_id.into();
|
||||
info.uploader_url =
|
||||
format!("https://www.youtube.com/channel/{browse_id}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content
|
||||
// .sectionListRenderer.contents[0].itemSectionRenderer.contents[0]
|
||||
// .playlistVideoListRenderer.contents[]
|
||||
let list_contents = body
|
||||
.get("contents")
|
||||
.and_then(|c| c.get("twoColumnBrowseResultsRenderer"))
|
||||
.and_then(|c| c.get("tabs"))
|
||||
.and_then(|t| t.as_array())
|
||||
.and_then(|tabs| tabs.first())
|
||||
.and_then(|t| t.get("tabRenderer"))
|
||||
.and_then(|t| t.get("content"))
|
||||
.and_then(|c| c.get("sectionListRenderer"))
|
||||
.and_then(|s| s.get("contents"))
|
||||
.and_then(|c| c.as_array())
|
||||
.and_then(|arr| arr.first())
|
||||
.and_then(|s| s.get("itemSectionRenderer"))
|
||||
.and_then(|i| i.get("contents"))
|
||||
.and_then(|c| c.as_array())
|
||||
.and_then(|arr| arr.first())
|
||||
.and_then(|s| s.get("playlistVideoListRenderer"))
|
||||
.and_then(|p| p.get("contents"))
|
||||
.and_then(|c| c.as_array());
|
||||
|
||||
if let Some(arr) = list_contents {
|
||||
for item in arr {
|
||||
if let Some(v) = item.get("playlistVideoRenderer") {
|
||||
if let Some(s) = parse_playlist_video_renderer(v) {
|
||||
info.videos.push(s);
|
||||
}
|
||||
} else if let Some(c) = item.get("continuationItemRenderer") {
|
||||
info.continuation_token = c
|
||||
.get("continuationEndpoint")
|
||||
.and_then(|e| e.get("continuationCommand"))
|
||||
.and_then(|c| c.get("token"))
|
||||
.and_then(|t| t.as_str())
|
||||
.map(String::from);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info
|
||||
}
|
||||
|
||||
fn parse_playlist_video_renderer(renderer: &Value) -> Option<StreamInfoItem> {
|
||||
let video_id = renderer.get("videoId")?.as_str()?.to_string();
|
||||
let title = renderer
|
||||
.get("title")
|
||||
.and_then(|t| t.get("runs"))
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|a| a.first())
|
||||
.and_then(|r| r.get("text"))
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let uploader_name = renderer
|
||||
.get("shortBylineText")
|
||||
.and_then(|s| s.get("runs"))
|
||||
.and_then(|r| r.as_array())
|
||||
.and_then(|a| a.first())
|
||||
.and_then(|r| r.get("text"))
|
||||
.and_then(|t| t.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let duration_seconds = renderer
|
||||
.get("lengthSeconds")
|
||||
.and_then(|s| s.as_str())
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
Some(StreamInfoItem {
|
||||
service_id: 0,
|
||||
url: format!("https://www.youtube.com/watch?v={video_id}"),
|
||||
name: title,
|
||||
thumbnails: Vec::new(),
|
||||
uploader_name,
|
||||
uploader_url: String::new(),
|
||||
uploader_id: String::new(),
|
||||
uploader_verified: false,
|
||||
duration_seconds,
|
||||
view_count: -1,
|
||||
upload_date_relative: String::new(),
|
||||
stream_type: Some(crate::stream::StreamType::VideoStream),
|
||||
short_description: String::new(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parses_basic_playlist_meta() {
|
||||
let body = json!({
|
||||
"metadata":{"playlistMetadataRenderer":{
|
||||
"title":"Coding music",
|
||||
"description":"For long sessions."
|
||||
}}
|
||||
});
|
||||
let info = parse_playlist_browse("PLxxx", &body);
|
||||
assert_eq!(info.name, "Coding music");
|
||||
assert_eq!(info.description, "For long sessions.");
|
||||
assert_eq!(info.playlist_id, "PLxxx");
|
||||
assert_eq!(info.url, "https://www.youtube.com/playlist?list=PLxxx");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_video_list_and_continuation() {
|
||||
let body = json!({
|
||||
"contents":{"twoColumnBrowseResultsRenderer":{"tabs":[{
|
||||
"tabRenderer":{"content":{"sectionListRenderer":{"contents":[{
|
||||
"itemSectionRenderer":{"contents":[{
|
||||
"playlistVideoListRenderer":{"contents":[
|
||||
{"playlistVideoRenderer":{
|
||||
"videoId":"abc",
|
||||
"title":{"runs":[{"text":"First track"}]},
|
||||
"shortBylineText":{"runs":[{"text":"NCS"}]},
|
||||
"lengthSeconds":"234"
|
||||
}},
|
||||
{"continuationItemRenderer":{
|
||||
"continuationEndpoint":{"continuationCommand":{
|
||||
"token":"OPAQUE_CONT_TOKEN"
|
||||
}}
|
||||
}}
|
||||
]}
|
||||
}]}
|
||||
}]}}}
|
||||
}]}}
|
||||
});
|
||||
let info = parse_playlist_browse("PLxxx", &body);
|
||||
assert_eq!(info.videos.len(), 1);
|
||||
assert_eq!(info.videos[0].name, "First track");
|
||||
assert_eq!(info.videos[0].uploader_name, "NCS");
|
||||
assert_eq!(info.videos[0].duration_seconds, 234);
|
||||
assert_eq!(info.continuation_token.as_deref(), Some("OPAQUE_CONT_TOKEN"));
|
||||
}
|
||||
}
|
||||
|
|
@ -39,11 +39,6 @@ pub struct SearchInfo {
|
|||
}
|
||||
|
||||
pub fn search(query: &str, filter: SearchFilter) -> Result<SearchInfo, ExtractionError> {
|
||||
if filter.uses_music_endpoint() {
|
||||
return Err(ExtractionError::Other(
|
||||
"music search filters route to WEB_REMIX — not implemented in this phase".into(),
|
||||
));
|
||||
}
|
||||
let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?;
|
||||
let localization = NewPipe::preferred_localization();
|
||||
let content_country = NewPipe::preferred_content_country();
|
||||
|
|
@ -158,7 +153,10 @@ fn extract_item_into(item: &Value, info: &mut SearchInfo) {
|
|||
// playlist extractors.
|
||||
}
|
||||
|
||||
pub(crate) mod test_helpers {
|
||||
/// Shared parsing helper for video-renderer JSON shape. Mis-named as
|
||||
/// `test_helpers` previously — it's production code called from
|
||||
/// `channel.rs`, not a test fixture. Renamed 2026-05-26.
|
||||
pub(crate) mod renderer_helpers {
|
||||
use super::*;
|
||||
pub fn video_renderer_to_item(renderer: &Value) -> Option<StreamInfoItem> {
|
||||
super::parse_video_renderer(renderer)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::exceptions::{ContentUnavailable, ExtractionError, NetworkError, ParsingError};
|
||||
use crate::exceptions::{ContentUnavailable, ExtractionError, ParsingError};
|
||||
use crate::image::{Image, ResolutionLevel};
|
||||
use crate::localization::{ContentCountry, Localization};
|
||||
use crate::newpipe::NewPipe;
|
||||
|
|
@ -744,9 +744,6 @@ fn push_video_dedup(list: &mut Vec<VideoStream>, candidate: VideoStream) {
|
|||
list.push(candidate);
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn _suppress_unused(_: MediaFormat, _: NetworkError) {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
// YoutubeSuggestionExtractor — search-as-you-type autocomplete.
|
||||
// Mirrors NPE services/youtube/extractors/YoutubeSuggestionExtractor.java.
|
||||
//
|
||||
// Endpoint:
|
||||
// GET https://suggestqueries-clients6.youtube.com/complete/search
|
||||
// ?client=youtube&ds=yt&gl=<cc>&q=<query>&xhr=t
|
||||
//
|
||||
// Returns a JSON array shaped like: `[query, [[suggestion, 0], ...], {}]`.
|
||||
// The XSSI prefix `)]}'\n` may NOT be present — NPE handles both cases.
|
||||
|
||||
use serde_json::Value;
|
||||
use url::form_urlencoded;
|
||||
|
||||
use crate::downloader::request::Request;
|
||||
use crate::exceptions::{ExtractionError, NetworkError, ParsingError};
|
||||
use crate::newpipe::NewPipe;
|
||||
|
||||
pub fn suggestions(query: &str) -> Result<Vec<String>, ExtractionError> {
|
||||
let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?;
|
||||
let cc = NewPipe::preferred_content_country();
|
||||
|
||||
let encoded: String = form_urlencoded::Serializer::new(String::new())
|
||||
.append_pair("client", "youtube")
|
||||
.append_pair("ds", "yt")
|
||||
.append_pair("gl", cc.country_code())
|
||||
.append_pair("q", query)
|
||||
.append_pair("xhr", "t")
|
||||
.finish();
|
||||
let url =
|
||||
format!("https://suggestqueries-clients6.youtube.com/complete/search?{encoded}");
|
||||
|
||||
let req = Request::get(&url).build();
|
||||
let resp = downloader.execute(req)?;
|
||||
if resp.response_code() != 200 {
|
||||
return Err(ExtractionError::Network(NetworkError::Transport(format!(
|
||||
"suggest HTTP {}",
|
||||
resp.response_code()
|
||||
))));
|
||||
}
|
||||
let body = resp.response_body();
|
||||
let stripped = body.strip_prefix(")]}'\n").unwrap_or(body);
|
||||
let parsed: Value = serde_json::from_str(stripped)
|
||||
.map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?;
|
||||
Ok(parse_suggestions(&parsed))
|
||||
}
|
||||
|
||||
pub fn parse_suggestions(value: &Value) -> Vec<String> {
|
||||
value
|
||||
.as_array()
|
||||
.and_then(|outer| outer.get(1))
|
||||
.and_then(|inner| inner.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|entry| {
|
||||
entry.as_array().and_then(|e| e.first()).and_then(|s| s.as_str())
|
||||
})
|
||||
.map(String::from)
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parses_typical_suggest_response() {
|
||||
let body = json!([
|
||||
"spek",
|
||||
[["spektrem", 0], ["spektrum", 0], ["spek tek", 0]],
|
||||
{}
|
||||
]);
|
||||
let out = parse_suggestions(&body);
|
||||
assert_eq!(out, vec!["spektrem", "spektrum", "spek tek"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_suggestions_array() {
|
||||
let body = json!(["q", []]);
|
||||
let out = parse_suggestions(&body);
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_malformed() {
|
||||
let body = json!({});
|
||||
assert!(parse_suggestions(&body).is_empty());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,186 +0,0 @@
|
|||
// Phase 4 offline tests for the stream-extraction parsing layer.
|
||||
//
|
||||
// Live YT extraction is gated behind the `online-tests` feature; these
|
||||
// tests exercise the JSON-walking and URL post-processing using a
|
||||
// hand-crafted player-response shaped like what YT actually returns
|
||||
// (videoDetails + streamingData.formats[] + streamingData.adaptiveFormats[]
|
||||
// + dashManifestUrl + captions). No network.
|
||||
|
||||
use serde_json::json;
|
||||
use strawcore::stream::DeliveryMethod;
|
||||
use strawcore::youtube::itag::MediaFormat;
|
||||
use strawcore::youtube::stream_extractor;
|
||||
|
||||
fn synthetic_android_response(video_id: &str) -> serde_json::Value {
|
||||
json!({
|
||||
"playabilityStatus": { "status": "OK" },
|
||||
"videoDetails": {
|
||||
"videoId": video_id,
|
||||
"title": "NCS Spektrem — Shine",
|
||||
"shortDescription": "Royalty-free music for streamers.",
|
||||
"lengthSeconds": "240",
|
||||
"viewCount": "42000000",
|
||||
"author": "NoCopyrightSounds",
|
||||
"channelId": "UC_aEa8K-EOJ3D6gOs7HcyNg",
|
||||
"isLive": false,
|
||||
"thumbnail": {
|
||||
"thumbnails": [
|
||||
{"url": "https://i.ytimg.com/vi/x/default.jpg", "width": 120, "height": 90},
|
||||
{"url": "https://i.ytimg.com/vi/x/maxresdefault.jpg", "width": 1920, "height": 1080}
|
||||
]
|
||||
}
|
||||
},
|
||||
"captions": {
|
||||
"playerCaptionsTracklistRenderer": {
|
||||
"captionTracks": [
|
||||
{
|
||||
"baseUrl": "https://www.youtube.com/api/timedtext?lang=en&v=x",
|
||||
"languageCode": "en",
|
||||
"name": {"simpleText": "English"},
|
||||
"kind": "asr"
|
||||
},
|
||||
{
|
||||
"baseUrl": "https://www.youtube.com/api/timedtext?lang=de&v=x",
|
||||
"languageCode": "de",
|
||||
"name": {"simpleText": "Deutsch"}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"streamingData": {
|
||||
"dashManifestUrl": "https://manifest.googlevideo.com/api/manifest/dash/foo/yes",
|
||||
"formats": [
|
||||
{
|
||||
"itag": 22,
|
||||
"url": "https://r1.googlevideo.com/videoplayback?expire=1&itag=22&c=ANDROID&n=ENCODEDNTOKEN",
|
||||
"mimeType": "video/mp4; codecs=\"avc1.64001F, mp4a.40.2\"",
|
||||
"bitrate": 1234567,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 30,
|
||||
"contentLength": "12345678"
|
||||
}
|
||||
],
|
||||
"adaptiveFormats": [
|
||||
{
|
||||
"itag": 140,
|
||||
"url": "https://r1.googlevideo.com/videoplayback?expire=1&itag=140&c=ANDROID&n=AUDIONTOKEN",
|
||||
"mimeType": "audio/mp4; codecs=\"mp4a.40.2\"",
|
||||
"averageBitrate": 128000,
|
||||
"contentLength": "4321000",
|
||||
"audioTrack": {
|
||||
"id": "en.4",
|
||||
"displayName": "English original",
|
||||
"audioIsDefault": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"itag": 251,
|
||||
"url": "https://r2.googlevideo.com/videoplayback?expire=1&itag=251&c=ANDROID&n=OPUSNTOKEN",
|
||||
"mimeType": "audio/webm; codecs=\"opus\"",
|
||||
"averageBitrate": 160000,
|
||||
"contentLength": "5555555"
|
||||
},
|
||||
{
|
||||
"itag": 137,
|
||||
"url": "https://r3.googlevideo.com/videoplayback?expire=1&itag=137&c=ANDROID&n=VIDEONTOKEN",
|
||||
"mimeType": "video/mp4; codecs=\"avc1.640028\"",
|
||||
"bitrate": 2500000,
|
||||
"width": 1920,
|
||||
"height": 1080,
|
||||
"fps": 30,
|
||||
"contentLength": "98765432"
|
||||
},
|
||||
{
|
||||
"itag": 999999,
|
||||
"url": "https://x/?itag=999999",
|
||||
"mimeType": "video/webm"
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Reaching the parsing fns requires a NewPipe::downloader configured,
|
||||
// because the orchestrator's first step is the live Android POST. We
|
||||
// don't want to hit the network in these tests, so the public
|
||||
// stream_info entry point doesn't run here. Instead we test the
|
||||
// behaviour-significant parsing helpers directly via the public test
|
||||
// surface that exposes them. Since those are currently private, we cover
|
||||
// the parsing layer through observable outputs by stitching a minimal
|
||||
// "post-android-call" mock path.
|
||||
//
|
||||
// We get there by checking that the synthetic response JSON shape is
|
||||
// what the orchestrator would see, and we verify the orchestrator's
|
||||
// individual helpers against it via the public `stream_extractor` module
|
||||
// — for the helpers that need NewPipe-init the smoke is implicitly
|
||||
// covered by Phase 1 + Phase 2 tests already.
|
||||
//
|
||||
// Concretely below: lightweight JSON-shape assertions that mirror what
|
||||
// populate_video_details / populate_streams would extract. If we change
|
||||
// the JSON wire-shape contract this catches it.
|
||||
|
||||
#[test]
|
||||
fn synthetic_response_has_expected_video_details_shape() {
|
||||
let r = synthetic_android_response("n4tK7LYFxI0");
|
||||
assert_eq!(r["videoDetails"]["videoId"], "n4tK7LYFxI0");
|
||||
assert_eq!(r["videoDetails"]["title"], "NCS Spektrem — Shine");
|
||||
assert_eq!(r["videoDetails"]["lengthSeconds"], "240");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_response_has_dash_manifest_url() {
|
||||
let r = synthetic_android_response("n4tK7LYFxI0");
|
||||
let url = r["streamingData"]["dashManifestUrl"].as_str().unwrap();
|
||||
assert!(url.starts_with("https://manifest.googlevideo.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthetic_response_has_progressive_and_adaptive_formats() {
|
||||
let r = synthetic_android_response("n4tK7LYFxI0");
|
||||
let progressive = r["streamingData"]["formats"].as_array().unwrap();
|
||||
assert_eq!(progressive.len(), 1);
|
||||
assert_eq!(progressive[0]["itag"], 22);
|
||||
|
||||
let adaptive = r["streamingData"]["adaptiveFormats"].as_array().unwrap();
|
||||
let itags: Vec<u64> = adaptive
|
||||
.iter()
|
||||
.map(|f| f["itag"].as_u64().unwrap())
|
||||
.collect();
|
||||
assert!(itags.contains(&140));
|
||||
assert!(itags.contains(&251));
|
||||
assert!(itags.contains(&137));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn options_default_disables_ios() {
|
||||
let opts = stream_extractor::ExtractOptions::default();
|
||||
assert!(!opts.fetch_ios_client);
|
||||
assert!(opts.android_streaming_pot.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_itags_lookup_ok() {
|
||||
use strawcore::youtube::itag::lookup;
|
||||
assert!(lookup(22).is_some()); // progressive 720p mp4
|
||||
assert!(lookup(140).is_some()); // m4a 128
|
||||
assert!(lookup(251).is_some()); // opus 160
|
||||
assert!(lookup(137).is_some()); // 1080p video-only mp4
|
||||
assert!(lookup(999999).is_none()); // unknown
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn known_itag_140_is_aac_128() {
|
||||
use strawcore::youtube::itag::{lookup, ItagType};
|
||||
let it = lookup(140).unwrap();
|
||||
assert_eq!(it.item_type, ItagType::Audio);
|
||||
assert_eq!(it.format, MediaFormat::M4A);
|
||||
assert_eq!(it.avg_bitrate_kbps, Some(128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delivery_method_progressive_vs_dash() {
|
||||
// Sanity that the enum is what the consumer expects to discriminate
|
||||
// (StraawApp's Media3 routing logic depends on this).
|
||||
assert_ne!(DeliveryMethod::Progressive, DeliveryMethod::Dash);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue