From d4000a9f9ac82b1b4be76b605d2185cae453325c Mon Sep 17 00:00:00 2001 From: Kayos Date: Tue, 26 May 2026 22:16:11 -0700 Subject: [PATCH] Cleanup: drop playlist + suggestion + dead client constants + suppress_unused stubs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 cruft audit punch list — mechanical deletes, no behavior change. Whole modules deleted (no wrapper consumer): * youtube/playlist_extractor.rs (297 LOC) — full playlist extraction * youtube/linkhandler/playlist.rs (81 LOC) — playlist URL parser * youtube/suggestion_extractor.rs (91 LOC) — search-as-you-type * tests/stream_phase4_offline.rs (186 LOC) — tautological test Dead pub fns + enum variants + constants: * WEB_REMIX_* constants (3) + WEB_MUSIC_ANALYTICS_* constants (3) * InnertubeClientRequestInfo::of_web_music_analytics_charts_client factory + its charts_client_omits_platform_and_screen test * SearchFilter::Music{Songs,Videos,Albums,Playlists,Artists} variants (5 of 9 cases) + uses_music_endpoint helper + the search_extractor 'music search not implemented' reject branch * Two #[allow(dead_code)] _suppress_unused stub fns and the imports they were keeping alive (std::sync::Arc in js/extractor.rs, NetworkError in stream_extractor.rs) Renamed: * search_extractor::test_helpers -> renderer_helpers. Mis-named: it's production code called from channel.rs, not a test fixture. potoken/ kept and documented as the designed Phase-5 extension point for YouTube bot-detection — wrapper's Android side hasn't registered a real provider yet, but the trait + global slot stay so when YT forces po_token universally the integration is one Kotlin patch away, not a Rust-side rewrite. ~580 LOC removed from production. Wrapper does not need to change. --- src/youtube/channel.rs | 2 +- src/youtube/client_request.rs | 32 --- src/youtube/constants.rs | 18 +- src/youtube/js/extractor.rs | 5 - src/youtube/linkhandler/mod.rs | 1 - src/youtube/linkhandler/playlist.rs | 81 -------- src/youtube/linkhandler/search.rs | 44 +---- src/youtube/mod.rs | 15 +- src/youtube/playlist_extractor.rs | 297 ---------------------------- src/youtube/search_extractor.rs | 10 +- src/youtube/stream_extractor.rs | 5 +- src/youtube/suggestion_extractor.rs | 91 --------- tests/stream_phase4_offline.rs | 186 ----------------- 13 files changed, 27 insertions(+), 760 deletions(-) delete mode 100644 src/youtube/linkhandler/playlist.rs delete mode 100644 src/youtube/playlist_extractor.rs delete mode 100644 src/youtube/suggestion_extractor.rs delete mode 100644 tests/stream_phase4_offline.rs diff --git a/src/youtube/channel.rs b/src/youtube/channel.rs index bd1fe03..a28e3ed 100644 --- a/src/youtube/channel.rs +++ b/src/youtube/channel.rs @@ -255,7 +255,7 @@ fn parse_videos_tab(body: &Value) -> Vec { continue; }; if let Some(vr) = content.get("videoRenderer") { - if let Some(item) = crate::youtube::search_extractor::test_helpers::video_renderer_to_item(vr) { + if let Some(item) = crate::youtube::search_extractor::renderer_helpers::video_renderer_to_item(vr) { out.push(item); } } else if let Some(lvm) = content.get("lockupViewModel") { diff --git a/src/youtube/client_request.rs b/src/youtube/client_request.rs index 96f3226..63f2547 100644 --- a/src/youtube/client_request.rs +++ b/src/youtube/client_request.rs @@ -73,23 +73,6 @@ impl InnertubeClientRequestInfo { } } - pub fn of_web_music_analytics_charts_client() -> Self { - // NPE deliberately omits clientScreen + platform for charts. - Self { - client_info: ClientInfo { - client_name: WEB_MUSIC_ANALYTICS_CLIENT_NAME.into(), - client_version: WEB_MUSIC_ANALYTICS_CLIENT_VERSION.into(), - client_id: WEB_MUSIC_ANALYTICS_CLIENT_ID.into(), - client_screen: None, - visitor_data: None, - }, - device_info: DeviceInfo { - android_sdk_version: -1, - ..Default::default() - }, - } - } - pub fn of_android_client() -> Self { Self { client_info: ClientInfo { @@ -292,21 +275,6 @@ mod tests { assert!(client.get("androidSdkVersion").is_none()); } - #[test] - fn charts_client_omits_platform_and_screen() { - let info = InnertubeClientRequestInfo::of_web_music_analytics_charts_client(); - let env = build_envelope( - &info, - &Localization::default(), - &ContentCountry::default(), - None, - ); - let client = &env["context"]["client"]; - assert_eq!(client["clientName"], "WEB_MUSIC_ANALYTICS"); - assert!(client.get("clientScreen").is_none()); - assert!(client.get("platform").is_none()); - } - #[test] fn embed_url_lands_in_third_party_block() { let info = InnertubeClientRequestInfo::of_web_embedded_player_client(); diff --git a/src/youtube/constants.rs b/src/youtube/constants.rs index d70e797..6673da8 100644 --- a/src/youtube/constants.rs +++ b/src/youtube/constants.rs @@ -1,8 +1,8 @@ -// ClientsConstants — mirrors NPE services/youtube/ClientsConstants.java. -// -// Six live InnerTube clients: WEB, WEB_EMBEDDED_PLAYER, WEB_MUSIC_ANALYTICS, -// WEB_REMIX, ANDROID, IOS. NPE's tree also mentions TVHTML5 + MWEB in -// comments — not actively used; skipped per audit Track A §1.3. +// InnerTube client constants. Active clients: WEB, WEB_EMBEDDED_PLAYER, +// ANDROID, IOS. WEB_REMIX (music) and WEB_MUSIC_ANALYTICS were ported +// but never wired through — dropped 2026-05-26 along with the music +// SearchFilter variants and the of_web_music_analytics_charts_client +// factory. pub const DESKTOP_CLIENT_PLATFORM: &str = "DESKTOP"; pub const MOBILE_CLIENT_PLATFORM: &str = "MOBILE"; @@ -13,18 +13,10 @@ pub const WEB_CLIENT_ID: &str = "1"; pub const WEB_CLIENT_NAME: &str = "WEB"; pub const WEB_HARDCODED_CLIENT_VERSION: &str = "2.20260120.01.00"; -pub const WEB_REMIX_CLIENT_ID: &str = "67"; -pub const WEB_REMIX_CLIENT_NAME: &str = "WEB_REMIX"; -pub const WEB_REMIX_HARDCODED_CLIENT_VERSION: &str = "1.20260121.03.00"; - pub const WEB_EMBEDDED_CLIENT_ID: &str = "56"; pub const WEB_EMBEDDED_CLIENT_NAME: &str = "WEB_EMBEDDED_PLAYER"; pub const WEB_EMBEDDED_CLIENT_VERSION: &str = "1.20260122.01.00"; -pub const WEB_MUSIC_ANALYTICS_CLIENT_ID: &str = "31"; -pub const WEB_MUSIC_ANALYTICS_CLIENT_NAME: &str = "WEB_MUSIC_ANALYTICS"; -pub const WEB_MUSIC_ANALYTICS_CLIENT_VERSION: &str = "2.0"; - // iPhone 15 Pro Max — chosen explicitly for 60fps tags per ItagItem.java:26 // note and the gist referenced in NPE source. pub const IOS_CLIENT_ID: &str = "5"; diff --git a/src/youtube/js/extractor.rs b/src/youtube/js/extractor.rs index fb9262c..ab3d81c 100644 --- a/src/youtube/js/extractor.rs +++ b/src/youtube/js/extractor.rs @@ -11,8 +11,6 @@ // Jsoup's attr-setter doesn't filter. Our walk does the same — iterate // every script tag, return first whose `src` contains `base.js`. -use std::sync::Arc; - use once_cell::sync::Lazy; use regex::Regex; @@ -137,9 +135,6 @@ pub fn extract_player_hash(url: &str) -> Option { RE.captures(url).and_then(|c| c.get(1)).map(|m| m.as_str().to_string()) } -#[allow(dead_code)] -fn _suppress_unused_arc_import(_: Arc) {} - #[cfg(test)] mod tests { use super::*; diff --git a/src/youtube/linkhandler/mod.rs b/src/youtube/linkhandler/mod.rs index 2af62a7..26d7404 100644 --- a/src/youtube/linkhandler/mod.rs +++ b/src/youtube/linkhandler/mod.rs @@ -7,7 +7,6 @@ // mirror list in NPE is dropped — Sulkta isn't an Invidious mirror. pub mod channel; -pub mod playlist; pub mod search; pub mod stream; diff --git a/src/youtube/linkhandler/playlist.rs b/src/youtube/linkhandler/playlist.rs deleted file mode 100644 index 00a58d7..0000000 --- a/src/youtube/linkhandler/playlist.rs +++ /dev/null @@ -1,81 +0,0 @@ -// YoutubePlaylistLinkHandlerFactory — accepts: -// * https://www.youtube.com/playlist?list= -// * https://www.youtube.com/watch?v=...&list= -// * https://music.youtube.com/playlist?list= -// -// YT playlist IDs prefix: -// * PL user-curated playlists -// * RD mix / radio -// * OLAK5uy_ album / single -// * LL liked-videos (private — won't extract anonymously) -// * WL watch-later (private) -// * UU uploads (auto-generated per channel) - -use url::Url; - -use crate::youtube::linkhandler::{host_is_youtube, LinkError}; - -pub fn extract_playlist_id(url_str: &str) -> Result { - let url = Url::parse(url_str) - .map_err(|e| LinkError::InvalidUrl(format!("{url_str}: {e}")))?; - let host = url - .host_str() - .ok_or_else(|| LinkError::InvalidUrl("no host".into()))?; - if !host_is_youtube(host) { - return Err(LinkError::UnsupportedHost(host.into())); - } - url.query_pairs() - .find(|(k, _)| k == "list") - .map(|(_, v)| v.into_owned()) - .filter(|s| !s.is_empty()) - .ok_or_else(|| LinkError::MissingId(url_str.into())) -} - -pub fn playlist_url(playlist_id: &str) -> String { - format!("https://www.youtube.com/playlist?list={playlist_id}") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn standalone_playlist() { - let id = extract_playlist_id( - "https://www.youtube.com/playlist?list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj", - ) - .unwrap(); - assert_eq!(id, "PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"); - } - - #[test] - fn watch_with_list() { - let id = extract_playlist_id( - "https://www.youtube.com/watch?v=dQw4w9WgXcQ&list=PLxxx", - ) - .unwrap(); - assert_eq!(id, "PLxxx"); - } - - #[test] - fn music_subdomain() { - let id = extract_playlist_id( - "https://music.youtube.com/playlist?list=OLAK5uy_kFooBar", - ) - .unwrap(); - assert_eq!(id, "OLAK5uy_kFooBar"); - } - - #[test] - fn rejects_no_list_param() { - let err = extract_playlist_id("https://www.youtube.com/watch?v=dQw4w9WgXcQ") - .unwrap_err(); - assert!(matches!(err, LinkError::MissingId(_))); - } - - #[test] - fn rejects_non_youtube_host() { - let err = extract_playlist_id("https://invidious.io/playlist?list=PLxxx").unwrap_err(); - assert!(matches!(err, LinkError::UnsupportedHost(_))); - } -} diff --git a/src/youtube/linkhandler/search.rs b/src/youtube/linkhandler/search.rs index 0cda6bf..23a963c 100644 --- a/src/youtube/linkhandler/search.rs +++ b/src/youtube/linkhandler/search.rs @@ -1,9 +1,11 @@ -// YoutubeSearchQueryHandlerFactory + search filters. Mirrors NPE -// YoutubeSearchQueryHandlerFactory.java + the filter params in -// YoutubeSearchExtractor.java. +// YoutubeSearchQueryHandlerFactory + search filters. Filter params are +// opaque base64 protobufs — NPE doesn't decode them, just sends the +// magic strings. We mirror that. See audit Track D §3. // -// Filter params are opaque base64 protobufs — NPE doesn't decode them, -// just sends the magic strings. We mirror that. See audit Track D §3. +// Music* filter variants were ported from NPE but never wired through +// (search_extractor rejected anything with `uses_music_endpoint()`). +// Dropped 2026-05-26 along with WEB_REMIX_* constants + the music +// charts client factory. use url::form_urlencoded; @@ -17,16 +19,6 @@ pub enum SearchFilter { Channels, /// Playlists only. Playlists, - /// "Music songs" filter — uses the WEB_REMIX path on music.youtube.com. - MusicSongs, - /// "Music videos" filter — also WEB_REMIX. - MusicVideos, - /// "Music albums" filter. - MusicAlbums, - /// "Music playlists" filter. - MusicPlaylists, - /// "Music artists" filter. - MusicArtists, } impl SearchFilter { @@ -38,24 +30,8 @@ impl SearchFilter { SearchFilter::Videos => Some("EgIQAfABAQ%3D%3D"), SearchFilter::Channels => Some("EgIQAvABAQ%3D%3D"), SearchFilter::Playlists => Some("EgIQA_ABAQ%3D%3D"), - SearchFilter::MusicSongs => Some("EgWKAQIIAWoMEA4QChADEAQQCRAF"), - SearchFilter::MusicVideos => Some("EgWKAQIQAWoMEA4QChADEAQQCRAF"), - SearchFilter::MusicAlbums => Some("EgWKAQIYAWoMEA4QChADEAQQCRAF"), - SearchFilter::MusicPlaylists => Some("EgeKAQQoAEABagwQDhAKEAMQBBAJEAU%3D"), - SearchFilter::MusicArtists => Some("EgWKAQIgAWoMEA4QChADEAQQCRAF"), } } - - pub fn uses_music_endpoint(&self) -> bool { - matches!( - self, - SearchFilter::MusicSongs - | SearchFilter::MusicVideos - | SearchFilter::MusicAlbums - | SearchFilter::MusicPlaylists - | SearchFilter::MusicArtists - ) - } } pub fn search_url(query: &str) -> String { @@ -81,12 +57,6 @@ mod tests { assert!(SearchFilter::Playlists.params().is_some()); } - #[test] - fn music_filters_route_to_music_endpoint() { - assert!(SearchFilter::MusicSongs.uses_music_endpoint()); - assert!(!SearchFilter::Videos.uses_music_endpoint()); - } - #[test] fn search_url_encodes_query() { assert_eq!( diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index ca60c83..0081c2c 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -1,7 +1,6 @@ -// YouTube service tree. Phase 2 landed the JS deobfuscator. Phase 3 adds -// the InnerTube client matrix, request envelope, parsing helpers, and the -// itag table. Phase 4+ will add the stream extractor, search, channel, -// playlist, kiosks. +// YouTube service tree: JS deobfuscator, the InnerTube client matrix + +// request envelope, parsing helpers, itag table, and the stream / search +// / channel extractors. pub mod channel; pub mod client_request; @@ -10,10 +9,14 @@ pub mod itag; pub mod js; pub mod linkhandler; pub mod parsing; -pub mod playlist_extractor; +/// PoToken bot-detection hook. `stream_extractor` reads the global +/// provider via `po_token_provider()`; the Android side hasn't +/// registered a real provider yet, so streams currently flow without +/// a po_token (works until YT requires one universally). This module +/// is the designed extension point — do not delete just because the +/// provider is currently no-op. pub mod potoken; pub mod search_extractor; pub mod stream_extractor; pub mod stream_helper; -pub mod suggestion_extractor; diff --git a/src/youtube/playlist_extractor.rs b/src/youtube/playlist_extractor.rs deleted file mode 100644 index ec6c317..0000000 --- a/src/youtube/playlist_extractor.rs +++ /dev/null @@ -1,297 +0,0 @@ -// YoutubePlaylistExtractor — mirrors NPE -// services/youtube/extractors/YoutubePlaylistExtractor.java. -// -// 2-POST pattern (audit Track D §7): -// 1. browseId="VL" → playlist metadata + first batch -// 2. continuation token → subsequent batches -// -// Body shape per call: build_desktop_envelope + add browseId (or -// continuation). Response walked to playlistVideoListRenderer.contents[] -// .playlistVideoRenderer. - -use serde_json::Value; - -use crate::downloader::request::Request; -use crate::exceptions::{ExtractionError, NetworkError, ParsingError}; -use crate::image::ImageSet; -use crate::newpipe::NewPipe; -use crate::stream::StreamInfoItem; -use crate::youtube::client_request::build_desktop_envelope; -use crate::youtube::constants::*; -use crate::youtube::parsing::{web_client_version, youtube_post_headers}; - -#[derive(Clone, Debug, Default)] -pub struct PlaylistInfo { - pub playlist_id: String, - pub url: String, - pub name: String, - pub description: String, - pub uploader_name: String, - pub uploader_url: String, - pub uploader_id: String, - pub thumbnails: ImageSet, - pub video_count: i64, - pub videos: Vec, - pub continuation_token: Option, -} - -pub fn playlist_info(playlist_id: &str) -> Result { - let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; - let localization = NewPipe::preferred_localization(); - let content_country = NewPipe::preferred_content_country(); - - let mut envelope = - build_desktop_envelope(&localization, &content_country, &web_client_version()); - if let Value::Object(ref mut map) = envelope { - map.insert( - "browseId".into(), - Value::String(format!("VL{playlist_id}")), - ); - } - let url = format!("{YOUTUBEI_V1_URL}browse{DISABLE_PRETTY_PRINT_PARAM}"); - let body = serde_json::to_vec(&envelope).map_err(|e| { - ExtractionError::Parsing(ParsingError::Invalid(format!("serialize: {e}"))) - })?; - let mut builder = Request::post(&url, body); - for (k, v) in youtube_post_headers() { - builder = builder.add_header(&k, &v); - } - let resp = downloader.execute(builder.build())?; - if resp.response_code() != 200 { - return Err(ExtractionError::Network(NetworkError::Transport(format!( - "browse HTTP {}", - resp.response_code() - )))); - } - let parsed: Value = serde_json::from_str(resp.response_body()) - .map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?; - Ok(parse_playlist_browse(playlist_id, &parsed)) -} - -pub fn parse_playlist_browse(playlist_id: &str, body: &Value) -> PlaylistInfo { - let mut info = PlaylistInfo { - playlist_id: playlist_id.into(), - url: format!("https://www.youtube.com/playlist?list={playlist_id}"), - ..PlaylistInfo::default() - }; - - // metadata.playlistMetadataRenderer.title / description - if let Some(meta) = body - .get("metadata") - .and_then(|m| m.get("playlistMetadataRenderer")) - { - if let Some(s) = meta.get("title").and_then(|v| v.as_str()) { - info.name = s.into(); - } - if let Some(s) = meta.get("description").and_then(|v| v.as_str()) { - info.description = s.into(); - } - } - - // sidebar.playlistSidebarRenderer.items[].playlistSidebarPrimaryInfoRenderer - // + playlistSidebarSecondaryInfoRenderer - if let Some(items) = body - .get("sidebar") - .and_then(|s| s.get("playlistSidebarRenderer")) - .and_then(|s| s.get("items")) - .and_then(|i| i.as_array()) - { - for item in items { - if let Some(primary) = item.get("playlistSidebarPrimaryInfoRenderer") { - if info.name.is_empty() { - if let Some(s) = primary - .get("title") - .and_then(|t| t.get("runs")) - .and_then(|r| r.as_array()) - .and_then(|a| a.first()) - .and_then(|r| r.get("text")) - .and_then(|t| t.as_str()) - { - info.name = s.into(); - } - } - // stats[1] (video count) — "1,234 videos" - if let Some(stats) = primary.get("stats").and_then(|s| s.as_array()) { - if let Some(count_text) = stats - .get(0) - .and_then(|s| s.get("runs")) - .and_then(|r| r.as_array()) - .and_then(|a| a.first()) - .and_then(|r| r.get("text")) - .and_then(|t| t.as_str()) - { - info.video_count = count_text - .replace(',', "") - .split_whitespace() - .next() - .and_then(|s| s.parse().ok()) - .unwrap_or(-1); - } - } - } - if let Some(secondary) = item.get("playlistSidebarSecondaryInfoRenderer") { - if let Some(owner) = secondary.get("videoOwner").and_then(|o| { - o.get("videoOwnerRenderer") - }) { - if let Some(s) = owner - .get("title") - .and_then(|t| t.get("runs")) - .and_then(|r| r.as_array()) - .and_then(|a| a.first()) - { - if let Some(name) = s.get("text").and_then(|t| t.as_str()) { - info.uploader_name = name.into(); - } - if let Some(endpoint) = s.get("navigationEndpoint") { - if let Some(browse_id) = endpoint - .get("browseEndpoint") - .and_then(|b| b.get("browseId")) - .and_then(|i| i.as_str()) - { - info.uploader_id = browse_id.into(); - info.uploader_url = - format!("https://www.youtube.com/channel/{browse_id}"); - } - } - } - } - } - } - } - - // contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content - // .sectionListRenderer.contents[0].itemSectionRenderer.contents[0] - // .playlistVideoListRenderer.contents[] - let list_contents = body - .get("contents") - .and_then(|c| c.get("twoColumnBrowseResultsRenderer")) - .and_then(|c| c.get("tabs")) - .and_then(|t| t.as_array()) - .and_then(|tabs| tabs.first()) - .and_then(|t| t.get("tabRenderer")) - .and_then(|t| t.get("content")) - .and_then(|c| c.get("sectionListRenderer")) - .and_then(|s| s.get("contents")) - .and_then(|c| c.as_array()) - .and_then(|arr| arr.first()) - .and_then(|s| s.get("itemSectionRenderer")) - .and_then(|i| i.get("contents")) - .and_then(|c| c.as_array()) - .and_then(|arr| arr.first()) - .and_then(|s| s.get("playlistVideoListRenderer")) - .and_then(|p| p.get("contents")) - .and_then(|c| c.as_array()); - - if let Some(arr) = list_contents { - for item in arr { - if let Some(v) = item.get("playlistVideoRenderer") { - if let Some(s) = parse_playlist_video_renderer(v) { - info.videos.push(s); - } - } else if let Some(c) = item.get("continuationItemRenderer") { - info.continuation_token = c - .get("continuationEndpoint") - .and_then(|e| e.get("continuationCommand")) - .and_then(|c| c.get("token")) - .and_then(|t| t.as_str()) - .map(String::from); - } - } - } - - info -} - -fn parse_playlist_video_renderer(renderer: &Value) -> Option { - let video_id = renderer.get("videoId")?.as_str()?.to_string(); - let title = renderer - .get("title") - .and_then(|t| t.get("runs")) - .and_then(|r| r.as_array()) - .and_then(|a| a.first()) - .and_then(|r| r.get("text")) - .and_then(|t| t.as_str()) - .unwrap_or("") - .to_string(); - let uploader_name = renderer - .get("shortBylineText") - .and_then(|s| s.get("runs")) - .and_then(|r| r.as_array()) - .and_then(|a| a.first()) - .and_then(|r| r.get("text")) - .and_then(|t| t.as_str()) - .unwrap_or("") - .to_string(); - let duration_seconds = renderer - .get("lengthSeconds") - .and_then(|s| s.as_str()) - .and_then(|s| s.parse().ok()) - .unwrap_or(0); - Some(StreamInfoItem { - service_id: 0, - url: format!("https://www.youtube.com/watch?v={video_id}"), - name: title, - thumbnails: Vec::new(), - uploader_name, - uploader_url: String::new(), - uploader_id: String::new(), - uploader_verified: false, - duration_seconds, - view_count: -1, - upload_date_relative: String::new(), - stream_type: Some(crate::stream::StreamType::VideoStream), - short_description: String::new(), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn parses_basic_playlist_meta() { - let body = json!({ - "metadata":{"playlistMetadataRenderer":{ - "title":"Coding music", - "description":"For long sessions." - }} - }); - let info = parse_playlist_browse("PLxxx", &body); - assert_eq!(info.name, "Coding music"); - assert_eq!(info.description, "For long sessions."); - assert_eq!(info.playlist_id, "PLxxx"); - assert_eq!(info.url, "https://www.youtube.com/playlist?list=PLxxx"); - } - - #[test] - fn parses_video_list_and_continuation() { - let body = json!({ - "contents":{"twoColumnBrowseResultsRenderer":{"tabs":[{ - "tabRenderer":{"content":{"sectionListRenderer":{"contents":[{ - "itemSectionRenderer":{"contents":[{ - "playlistVideoListRenderer":{"contents":[ - {"playlistVideoRenderer":{ - "videoId":"abc", - "title":{"runs":[{"text":"First track"}]}, - "shortBylineText":{"runs":[{"text":"NCS"}]}, - "lengthSeconds":"234" - }}, - {"continuationItemRenderer":{ - "continuationEndpoint":{"continuationCommand":{ - "token":"OPAQUE_CONT_TOKEN" - }} - }} - ]} - }]} - }]}}} - }]}} - }); - let info = parse_playlist_browse("PLxxx", &body); - assert_eq!(info.videos.len(), 1); - assert_eq!(info.videos[0].name, "First track"); - assert_eq!(info.videos[0].uploader_name, "NCS"); - assert_eq!(info.videos[0].duration_seconds, 234); - assert_eq!(info.continuation_token.as_deref(), Some("OPAQUE_CONT_TOKEN")); - } -} diff --git a/src/youtube/search_extractor.rs b/src/youtube/search_extractor.rs index 315338a..b50b280 100644 --- a/src/youtube/search_extractor.rs +++ b/src/youtube/search_extractor.rs @@ -39,11 +39,6 @@ pub struct SearchInfo { } pub fn search(query: &str, filter: SearchFilter) -> Result { - if filter.uses_music_endpoint() { - return Err(ExtractionError::Other( - "music search filters route to WEB_REMIX — not implemented in this phase".into(), - )); - } let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; let localization = NewPipe::preferred_localization(); let content_country = NewPipe::preferred_content_country(); @@ -158,7 +153,10 @@ fn extract_item_into(item: &Value, info: &mut SearchInfo) { // playlist extractors. } -pub(crate) mod test_helpers { +/// Shared parsing helper for video-renderer JSON shape. Mis-named as +/// `test_helpers` previously — it's production code called from +/// `channel.rs`, not a test fixture. Renamed 2026-05-26. +pub(crate) mod renderer_helpers { use super::*; pub fn video_renderer_to_item(renderer: &Value) -> Option { super::parse_video_renderer(renderer) diff --git a/src/youtube/stream_extractor.rs b/src/youtube/stream_extractor.rs index ccd6cad..f24670e 100644 --- a/src/youtube/stream_extractor.rs +++ b/src/youtube/stream_extractor.rs @@ -23,7 +23,7 @@ use serde_json::Value; -use crate::exceptions::{ContentUnavailable, ExtractionError, NetworkError, ParsingError}; +use crate::exceptions::{ContentUnavailable, ExtractionError, ParsingError}; use crate::image::{Image, ResolutionLevel}; use crate::localization::{ContentCountry, Localization}; use crate::newpipe::NewPipe; @@ -744,9 +744,6 @@ fn push_video_dedup(list: &mut Vec, candidate: VideoStream) { list.push(candidate); } -#[allow(dead_code)] -fn _suppress_unused(_: MediaFormat, _: NetworkError) {} - #[cfg(test)] mod tests { use super::*; diff --git a/src/youtube/suggestion_extractor.rs b/src/youtube/suggestion_extractor.rs deleted file mode 100644 index edc0e0b..0000000 --- a/src/youtube/suggestion_extractor.rs +++ /dev/null @@ -1,91 +0,0 @@ -// YoutubeSuggestionExtractor — search-as-you-type autocomplete. -// Mirrors NPE services/youtube/extractors/YoutubeSuggestionExtractor.java. -// -// Endpoint: -// GET https://suggestqueries-clients6.youtube.com/complete/search -// ?client=youtube&ds=yt&gl=&q=&xhr=t -// -// Returns a JSON array shaped like: `[query, [[suggestion, 0], ...], {}]`. -// The XSSI prefix `)]}'\n` may NOT be present — NPE handles both cases. - -use serde_json::Value; -use url::form_urlencoded; - -use crate::downloader::request::Request; -use crate::exceptions::{ExtractionError, NetworkError, ParsingError}; -use crate::newpipe::NewPipe; - -pub fn suggestions(query: &str) -> Result, ExtractionError> { - let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; - let cc = NewPipe::preferred_content_country(); - - let encoded: String = form_urlencoded::Serializer::new(String::new()) - .append_pair("client", "youtube") - .append_pair("ds", "yt") - .append_pair("gl", cc.country_code()) - .append_pair("q", query) - .append_pair("xhr", "t") - .finish(); - let url = - format!("https://suggestqueries-clients6.youtube.com/complete/search?{encoded}"); - - let req = Request::get(&url).build(); - let resp = downloader.execute(req)?; - if resp.response_code() != 200 { - return Err(ExtractionError::Network(NetworkError::Transport(format!( - "suggest HTTP {}", - resp.response_code() - )))); - } - let body = resp.response_body(); - let stripped = body.strip_prefix(")]}'\n").unwrap_or(body); - let parsed: Value = serde_json::from_str(stripped) - .map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?; - Ok(parse_suggestions(&parsed)) -} - -pub fn parse_suggestions(value: &Value) -> Vec { - value - .as_array() - .and_then(|outer| outer.get(1)) - .and_then(|inner| inner.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|entry| { - entry.as_array().and_then(|e| e.first()).and_then(|s| s.as_str()) - }) - .map(String::from) - .collect() - }) - .unwrap_or_default() -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn parses_typical_suggest_response() { - let body = json!([ - "spek", - [["spektrem", 0], ["spektrum", 0], ["spek tek", 0]], - {} - ]); - let out = parse_suggestions(&body); - assert_eq!(out, vec!["spektrem", "spektrum", "spek tek"]); - } - - #[test] - fn empty_suggestions_array() { - let body = json!(["q", []]); - let out = parse_suggestions(&body); - assert!(out.is_empty()); - } - - #[test] - fn handles_malformed() { - let body = json!({}); - assert!(parse_suggestions(&body).is_empty()); - } -} diff --git a/tests/stream_phase4_offline.rs b/tests/stream_phase4_offline.rs deleted file mode 100644 index 8c28609..0000000 --- a/tests/stream_phase4_offline.rs +++ /dev/null @@ -1,186 +0,0 @@ -// Phase 4 offline tests for the stream-extraction parsing layer. -// -// Live YT extraction is gated behind the `online-tests` feature; these -// tests exercise the JSON-walking and URL post-processing using a -// hand-crafted player-response shaped like what YT actually returns -// (videoDetails + streamingData.formats[] + streamingData.adaptiveFormats[] -// + dashManifestUrl + captions). No network. - -use serde_json::json; -use strawcore::stream::DeliveryMethod; -use strawcore::youtube::itag::MediaFormat; -use strawcore::youtube::stream_extractor; - -fn synthetic_android_response(video_id: &str) -> serde_json::Value { - json!({ - "playabilityStatus": { "status": "OK" }, - "videoDetails": { - "videoId": video_id, - "title": "NCS Spektrem — Shine", - "shortDescription": "Royalty-free music for streamers.", - "lengthSeconds": "240", - "viewCount": "42000000", - "author": "NoCopyrightSounds", - "channelId": "UC_aEa8K-EOJ3D6gOs7HcyNg", - "isLive": false, - "thumbnail": { - "thumbnails": [ - {"url": "https://i.ytimg.com/vi/x/default.jpg", "width": 120, "height": 90}, - {"url": "https://i.ytimg.com/vi/x/maxresdefault.jpg", "width": 1920, "height": 1080} - ] - } - }, - "captions": { - "playerCaptionsTracklistRenderer": { - "captionTracks": [ - { - "baseUrl": "https://www.youtube.com/api/timedtext?lang=en&v=x", - "languageCode": "en", - "name": {"simpleText": "English"}, - "kind": "asr" - }, - { - "baseUrl": "https://www.youtube.com/api/timedtext?lang=de&v=x", - "languageCode": "de", - "name": {"simpleText": "Deutsch"} - } - ] - } - }, - "streamingData": { - "dashManifestUrl": "https://manifest.googlevideo.com/api/manifest/dash/foo/yes", - "formats": [ - { - "itag": 22, - "url": "https://r1.googlevideo.com/videoplayback?expire=1&itag=22&c=ANDROID&n=ENCODEDNTOKEN", - "mimeType": "video/mp4; codecs=\"avc1.64001F, mp4a.40.2\"", - "bitrate": 1234567, - "width": 1280, - "height": 720, - "fps": 30, - "contentLength": "12345678" - } - ], - "adaptiveFormats": [ - { - "itag": 140, - "url": "https://r1.googlevideo.com/videoplayback?expire=1&itag=140&c=ANDROID&n=AUDIONTOKEN", - "mimeType": "audio/mp4; codecs=\"mp4a.40.2\"", - "averageBitrate": 128000, - "contentLength": "4321000", - "audioTrack": { - "id": "en.4", - "displayName": "English original", - "audioIsDefault": true - } - }, - { - "itag": 251, - "url": "https://r2.googlevideo.com/videoplayback?expire=1&itag=251&c=ANDROID&n=OPUSNTOKEN", - "mimeType": "audio/webm; codecs=\"opus\"", - "averageBitrate": 160000, - "contentLength": "5555555" - }, - { - "itag": 137, - "url": "https://r3.googlevideo.com/videoplayback?expire=1&itag=137&c=ANDROID&n=VIDEONTOKEN", - "mimeType": "video/mp4; codecs=\"avc1.640028\"", - "bitrate": 2500000, - "width": 1920, - "height": 1080, - "fps": 30, - "contentLength": "98765432" - }, - { - "itag": 999999, - "url": "https://x/?itag=999999", - "mimeType": "video/webm" - } - ] - } - }) -} - -// Reaching the parsing fns requires a NewPipe::downloader configured, -// because the orchestrator's first step is the live Android POST. We -// don't want to hit the network in these tests, so the public -// stream_info entry point doesn't run here. Instead we test the -// behaviour-significant parsing helpers directly via the public test -// surface that exposes them. Since those are currently private, we cover -// the parsing layer through observable outputs by stitching a minimal -// "post-android-call" mock path. -// -// We get there by checking that the synthetic response JSON shape is -// what the orchestrator would see, and we verify the orchestrator's -// individual helpers against it via the public `stream_extractor` module -// — for the helpers that need NewPipe-init the smoke is implicitly -// covered by Phase 1 + Phase 2 tests already. -// -// Concretely below: lightweight JSON-shape assertions that mirror what -// populate_video_details / populate_streams would extract. If we change -// the JSON wire-shape contract this catches it. - -#[test] -fn synthetic_response_has_expected_video_details_shape() { - let r = synthetic_android_response("n4tK7LYFxI0"); - assert_eq!(r["videoDetails"]["videoId"], "n4tK7LYFxI0"); - assert_eq!(r["videoDetails"]["title"], "NCS Spektrem — Shine"); - assert_eq!(r["videoDetails"]["lengthSeconds"], "240"); -} - -#[test] -fn synthetic_response_has_dash_manifest_url() { - let r = synthetic_android_response("n4tK7LYFxI0"); - let url = r["streamingData"]["dashManifestUrl"].as_str().unwrap(); - assert!(url.starts_with("https://manifest.googlevideo.com")); -} - -#[test] -fn synthetic_response_has_progressive_and_adaptive_formats() { - let r = synthetic_android_response("n4tK7LYFxI0"); - let progressive = r["streamingData"]["formats"].as_array().unwrap(); - assert_eq!(progressive.len(), 1); - assert_eq!(progressive[0]["itag"], 22); - - let adaptive = r["streamingData"]["adaptiveFormats"].as_array().unwrap(); - let itags: Vec = adaptive - .iter() - .map(|f| f["itag"].as_u64().unwrap()) - .collect(); - assert!(itags.contains(&140)); - assert!(itags.contains(&251)); - assert!(itags.contains(&137)); -} - -#[test] -fn options_default_disables_ios() { - let opts = stream_extractor::ExtractOptions::default(); - assert!(!opts.fetch_ios_client); - assert!(opts.android_streaming_pot.is_none()); -} - -#[test] -fn known_itags_lookup_ok() { - use strawcore::youtube::itag::lookup; - assert!(lookup(22).is_some()); // progressive 720p mp4 - assert!(lookup(140).is_some()); // m4a 128 - assert!(lookup(251).is_some()); // opus 160 - assert!(lookup(137).is_some()); // 1080p video-only mp4 - assert!(lookup(999999).is_none()); // unknown -} - -#[test] -fn known_itag_140_is_aac_128() { - use strawcore::youtube::itag::{lookup, ItagType}; - let it = lookup(140).unwrap(); - assert_eq!(it.item_type, ItagType::Audio); - assert_eq!(it.format, MediaFormat::M4A); - assert_eq!(it.avg_bitrate_kbps, Some(128)); -} - -#[test] -fn delivery_method_progressive_vs_dash() { - // Sanity that the enum is what the consumer expects to discriminate - // (StraawApp's Media3 routing logic depends on this). - assert_ne!(DeliveryMethod::Progressive, DeliveryMethod::Dash); -}