diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index dcb7c77..bd1ca76 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -235,7 +235,6 @@ fn map_artist_page( } } - mapper.check_unknown()?; let mut mapped = mapper.group_items(); static WIKIPEDIA_REGEX: Lazy = @@ -332,7 +331,6 @@ impl MapResponse> for response::MusicArtistAlbums { mapper.map_response(grid.grid_renderer.items); } - mapper.check_unknown()?; let mapped = mapper.group_items(); Ok(MapResult { diff --git a/src/client/music_charts.rs b/src/client/music_charts.rs index 7fc5b5a..27ac005 100644 --- a/src/client/music_charts.rs +++ b/src/client/music_charts.rs @@ -98,6 +98,7 @@ impl MapResponse for response::MusicCharts { h.music_carousel_shelf_basic_header_renderer .more_content_button .and_then(|btn| btn.button_renderer.navigation_endpoint.music_page()) + .map(|mp| (mp.typ, mp.id)) }) { Some((MusicPageType::Playlist, id)) => { // Top music videos (first shelf with associated playlist) @@ -120,10 +121,6 @@ impl MapResponse for response::MusicCharts { response::music_charts::ItemSection::None => {} }); - mapper_top.check_unknown()?; - mapper_trending.check_unknown()?; - mapper_other.check_unknown()?; - let mapped_top = mapper_top.conv_items::(); let mut mapped_trending = mapper_trending.conv_items::(); let mut mapped_other = mapper_other.group_items(); diff --git a/src/client/music_details.rs b/src/client/music_details.rs index b6d9361..36d5ddd 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -387,9 +387,6 @@ impl MapResponse for response::MusicRelated { _ => {} }); - mapper.check_unknown()?; - mapper_tracks.check_unknown()?; - let mapped_tracks = mapper_tracks.conv_items(); let mut mapped = mapper.group_items(); diff --git a/src/client/music_new.rs b/src/client/music_new.rs index dde4d2c..39914bb 100644 --- a/src/client/music_new.rs +++ b/src/client/music_new.rs @@ -75,7 +75,6 @@ impl MapResponse> for response::MusicNew { let mut mapper = MusicListMapper::new(lang); mapper.map_response(items); - mapper.check_unknown()?; Ok(mapper.conv_items()) } diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index ecf76dc..49350fb 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -174,7 +174,6 @@ impl MapResponse for response::MusicPlaylist { let mut mapper = MusicListMapper::new(lang); mapper.map_response(shelf.contents); - mapper.check_unknown()?; let map_res = mapper.conv_items(); let ctoken = shelf diff --git a/src/client/music_search.rs b/src/client/music_search.rs index 309bc1e..7d42842 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -266,7 +266,6 @@ impl MapResponse for response::MusicSearch { response::music_search::ItemSection::None => {} }); - mapper.check_unknown()?; let map_res = mapper.group_items(); Ok(MapResult { @@ -325,7 +324,6 @@ impl MapResponse> for response::MusicSearc response::music_search::ItemSection::None => {} }); - mapper.check_unknown()?; let map_res = mapper.conv_items(); Ok(MapResult { @@ -371,7 +369,6 @@ impl MapResponse for response::MusicSearchSuggestion { } } - mapper.check_unknown()?; let map_res = mapper.conv_items(); Ok(MapResult { diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index 62dc45a..95ab12e 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -2,14 +2,13 @@ use serde::Deserialize; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; use crate::{ - error::ExtractionError, model::{ self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, }, param::Language, serializer::{ - text::{Text, TextComponents}, + text::{Text, TextComponent, TextComponents}, MapResult, }, util::{self, dictionary}, @@ -17,7 +16,7 @@ use crate::{ use super::{ url_endpoint::{ - BrowseEndpointWrap, MusicPageType, MusicVideoType, NavigationEndpoint, PageType, + BrowseEndpointWrap, MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType, }, ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap, }; @@ -434,8 +433,6 @@ pub(crate) struct MusicListMapper { search_suggestion: bool, items: Vec, warnings: Vec, - /// True if unknown items were mapped - has_unknown: bool, } #[derive(Debug)] @@ -456,7 +453,6 @@ impl MusicListMapper { search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - has_unknown: false, } } @@ -469,7 +465,6 @@ impl MusicListMapper { search_suggestion: true, items: Vec::new(), warnings: Vec::new(), - has_unknown: false, } } @@ -483,7 +478,6 @@ impl MusicListMapper { search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - has_unknown: false, } } @@ -497,7 +491,6 @@ impl MusicListMapper { search_suggestion: false, items: Vec::new(), warnings: Vec::new(), - has_unknown: false, } } @@ -545,55 +538,44 @@ impl MusicListMapper { .thumbnails .first(); - let pt_id = item + let music_page = item .navigation_endpoint .and_then(NavigationEndpoint::music_page) .or_else(|| { c1.and_then(|c1| { - c1.renderer.text.0.into_iter().next().and_then(|t| match t { - crate::serializer::text::TextComponent::Video { - video_id, vtype, .. - } => Some((MusicPageType::Track { vtype }, video_id)), - crate::serializer::text::TextComponent::Browse { - page_type, - browse_id, - .. - } => Some((page_type.into(), browse_id)), - _ => None, - }) + c1.renderer + .text + .0 + .into_iter() + .next() + .and_then(TextComponent::music_page) }) }) .or_else(|| { - item.playlist_item_data.map(|d| { - ( - MusicPageType::Track { - vtype: MusicVideoType::from_is_video( - self.album.is_none() - && !first_tn - .map(|tn| tn.height == tn.width) - .unwrap_or_default(), - ), - }, - d.video_id, - ) + item.playlist_item_data.map(|d| MusicPage { + id: d.video_id, + typ: MusicPageType::Track { + vtype: MusicVideoType::from_is_video( + self.album.is_none() + && !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(), + ), + }, }) }) .or_else(|| { first_tn.and_then(|tn| { - util::video_id_from_thumbnail_url(&tn.url).map(|id| { - ( - MusicPageType::Track { - vtype: MusicVideoType::from_is_video( - self.album.is_none() && tn.width != tn.height, - ), - }, - id, - ) + util::video_id_from_thumbnail_url(&tn.url).map(|id| MusicPage { + id, + typ: MusicPageType::Track { + vtype: MusicVideoType::from_is_video( + self.album.is_none() && tn.width != tn.height, + ), + }, }) }) }); - match pt_id { + match music_page.map(|mp| (mp.typ, mp.id)) { // Track Some((MusicPageType::Track { vtype }, id)) => { let title = title.ok_or_else(|| format!("track {id}: could not get title"))?; @@ -852,10 +834,6 @@ impl MusicListMapper { } // Tracks were already handled above MusicPageType::Track { .. } => unreachable!(), - MusicPageType::Unknown => { - self.has_unknown = true; - Ok(None) - } } } None => { @@ -875,12 +853,12 @@ impl MusicListMapper { let subtitle_p2 = subtitle_parts.next(); match item.navigation_endpoint.music_page() { - Some((page_type, id)) => match page_type { + Some(music_page) => match music_page.typ { MusicPageType::Track { vtype } => { let (artists, by_va) = map_artists(subtitle_p1); self.items.push(MusicItem::Track(TrackItem { - id, + id: music_page.id, name: item.title, duration: None, cover: item.thumbnail_renderer.into(), @@ -910,7 +888,7 @@ impl MusicListMapper { }); self.items.push(MusicItem::Artist(ArtistItem { - id, + id: music_page.id, name: item.title, avatar: item.thumbnail_renderer.into(), subscriber_count, @@ -947,12 +925,15 @@ impl MusicListMapper { (Vec::new(), true) } _ => { - return Err(format!("could not parse subtitle of album {id}")); + return Err(format!( + "could not parse subtitle of album {}", + music_page.id + )); } }; self.items.push(MusicItem::Album(AlbumItem { - id, + id: music_page.id, name: item.title, cover: item.thumbnail_renderer.into(), artist_id: artists.first().and_then(|a| a.id.clone()), @@ -974,7 +955,7 @@ impl MusicListMapper { .and_then(|p| p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok())); self.items.push(MusicItem::Playlist(MusicPlaylistItem { - id, + id: music_page.id, name: item.title, thumbnail: item.thumbnail_renderer.into(), channel, @@ -984,10 +965,6 @@ impl MusicListMapper { Ok(Some(MusicItemType::Playlist)) } MusicPageType::None => Ok(None), - MusicPageType::Unknown => { - self.has_unknown = true; - Ok(None) - } }, None => Err("could not determine item type".to_owned()), } @@ -1009,7 +986,7 @@ impl MusicListMapper { let subtitle_p4 = subtitle_parts.next(); let item_type = match card.on_tap.music_page() { - Some((page_type, id)) => match page_type { + Some(music_page) => match music_page.typ { MusicPageType::Artist => { let subscriber_count = subtitle_p2.and_then(|p| { util::parse_large_numstr_or_warn( @@ -1020,7 +997,7 @@ impl MusicListMapper { }); self.items.push(MusicItem::Artist(ArtistItem { - id, + id: music_page.id, name: card.title, avatar: card.thumbnail.into(), subscriber_count, @@ -1034,7 +1011,7 @@ impl MusicListMapper { .unwrap_or_default(); self.items.push(MusicItem::Album(AlbumItem { - id, + id: music_page.id, name: card.title, cover: card.thumbnail.into(), artist_id: artists.first().and_then(|a| a.id.clone()), @@ -1050,7 +1027,7 @@ impl MusicListMapper { let (artists, by_va) = map_artists(subtitle_p3); self.items.push(MusicItem::Track(TrackItem { - id, + id: music_page.id, name: card.title, duration: None, cover: card.thumbnail.into(), @@ -1087,7 +1064,7 @@ impl MusicListMapper { }; self.items.push(MusicItem::Track(TrackItem { - id, + id: music_page.id, name: card.title, duration, cover: card.thumbnail.into(), @@ -1113,7 +1090,7 @@ impl MusicListMapper { subtitle_p3.and_then(|p| util::parse_numeric(p.first_str()).ok()); self.items.push(MusicItem::Playlist(MusicPlaylistItem { - id, + id: music_page.id, name: card.title, thumbnail: card.thumbnail.into(), channel, @@ -1123,10 +1100,6 @@ impl MusicListMapper { Some(MusicItemType::Playlist) } MusicPageType::None => None, - MusicPageType::Unknown => { - self.has_unknown = true; - None - } }, None => { self.warnings @@ -1201,20 +1174,6 @@ impl MusicListMapper { warnings: self.warnings, } } - - /// Sometimes the YT Music API returns responses containing unknown items. - /// - /// In this case, the response data is likely missing some fields, which leads to - /// parsing errors and wrong data being extracted. - /// - /// Therefore it is safest to discard such responses and retry the request. - pub fn check_unknown(&self) -> Result<(), ExtractionError> { - if self.has_unknown { - Err(ExtractionError::InvalidData("unknown YTM items".into())) - } else { - Ok(()) - } - } } /// Map TextComponents containing artist names to a list of artists and a 'Various Artists' flag diff --git a/src/client/response/url_endpoint.rs b/src/client/response/url_endpoint.rs index fb629fb..3bfbc35 100644 --- a/src/client/response/url_endpoint.rs +++ b/src/client/response/url_endpoint.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use serde_with::{serde_as, DefaultOnError}; -use crate::model::UrlTarget; +use crate::{model::UrlTarget, util}; /// navigation/resolve_url response model #[derive(Debug, Deserialize)] @@ -185,6 +185,10 @@ pub(crate) enum PageType { Channel, #[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")] Playlist, + #[serde(rename = "MUSIC_PAGE_TYPE_PODCAST_SHOW_DETAIL_PAGE")] + Podcast, + #[serde(rename = "MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE")] + Episode, #[default] Unknown, } @@ -195,6 +199,13 @@ impl PageType { PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }), PageType::Album => Some(UrlTarget::Album { id }), PageType::Playlist => Some(UrlTarget::Playlist { id }), + PageType::Podcast => Some(UrlTarget::Playlist { + id: util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX), + }), + PageType::Episode => Some(UrlTarget::Video { + id: util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX), + start_time: 0, + }), PageType::Unknown => None, } } @@ -206,7 +217,6 @@ pub(crate) enum MusicPageType { Album, Playlist, Track { vtype: MusicVideoType }, - Unknown, None, } @@ -215,16 +225,40 @@ impl From for MusicPageType { match t { PageType::Artist => MusicPageType::Artist, PageType::Album => MusicPageType::Album, - PageType::Playlist => MusicPageType::Playlist, - PageType::Channel => MusicPageType::None, - PageType::Unknown => MusicPageType::Unknown, + PageType::Playlist | PageType::Podcast => MusicPageType::Playlist, + PageType::Channel | PageType::Unknown => MusicPageType::None, + PageType::Episode => MusicPageType::Track { + vtype: MusicVideoType::Episode, + }, + } + } +} + +pub(crate) struct MusicPage { + pub id: String, + pub typ: MusicPageType, +} + +impl MusicPage { + /// Create a new MusicPage object, applying the required ID fixes when + /// mapping a browse link + pub fn from_browse(mut id: String, typ: PageType) -> Self { + if typ == PageType::Podcast { + id = util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX); + } else if typ == PageType::Episode && id.len() == 15 { + id = util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX); + } + + Self { + id, + typ: typ.into(), } } } impl NavigationEndpoint { /// Get the YouTube Music page and id from a browse/watch endpoint - pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> { + pub(crate) fn music_page(self) -> Option { match self { NavigationEndpoint::Watch { watch_endpoint } => { if watch_endpoint @@ -233,17 +267,20 @@ impl NavigationEndpoint { .unwrap_or_default() { // Genre radios (e.g. "pop radio") will be skipped - Some((MusicPageType::None, watch_endpoint.video_id)) + Some(MusicPage { + id: watch_endpoint.video_id, + typ: MusicPageType::None, + }) } else { - Some(( - MusicPageType::Track { + Some(MusicPage { + id: watch_endpoint.video_id, + typ: MusicPageType::Track { vtype: watch_endpoint .watch_endpoint_music_supported_configs .watch_endpoint_music_config .music_video_type, }, - watch_endpoint.video_id, - )) + }) } } NavigationEndpoint::Browse { @@ -251,9 +288,9 @@ impl NavigationEndpoint { } => browse_endpoint .browse_endpoint_context_supported_configs .map(|config| { - ( - config.browse_endpoint_context_music_config.page_type.into(), + MusicPage::from_browse( browse_endpoint.browse_id, + config.browse_endpoint_context_music_config.page_type, ) }), NavigationEndpoint::Url { .. } => None, diff --git a/src/serializer/text.rs b/src/serializer/text.rs index 4d628af..90ad772 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -6,7 +6,9 @@ use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DeserializeAs, VecSkipError}; use crate::{ - client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType}, + client::response::url_endpoint::{ + MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType, + }, model::UrlTarget, util, }; @@ -419,6 +421,23 @@ impl TextComponent { | TextComponent::Text { text } => text, } } + + pub fn music_page(self) -> Option { + match self { + TextComponent::Video { + video_id, vtype, .. + } => Some(MusicPage { + id: video_id, + typ: MusicPageType::Track { vtype }, + }), + TextComponent::Browse { + page_type, + browse_id, + .. + } => Some(MusicPage::from_browse(browse_id, page_type)), + _ => None, + } + } } impl From for String { diff --git a/src/util/mod.rs b/src/util/mod.rs index be7c50c..9de438d 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -41,6 +41,8 @@ pub const DOT_SEPARATOR: &str = " • "; pub const VARIOUS_ARTISTS: &str = "Various Artists"; pub const PLAYLIST_ID_ALBUM_PREFIX: &str = "OLAK"; pub const ARTIST_DISCOGRAPHY_PREFIX: &str = "MPAD"; +pub const PODCAST_PLAYLIST_PREFIX: &str = "MPSP"; +pub const PODCAST_EPISODE_PREFIX: &str = "MPED"; const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -474,6 +476,11 @@ pub fn country_from_name(name: &str) -> Option { .map(|i| COUNTRIES[i]) } +/// Strip prefix from string if presend +pub fn strip_prefix(s: &str, prefix: &str) -> String { + s.strip_prefix(prefix).unwrap_or(s).to_string() +} + /// An iterator over the chars in a string (in str format) pub struct SplitChar<'a> { txt: &'a str, diff --git a/tests/youtube.rs b/tests/youtube.rs index 687c010..6a34b15 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1664,7 +1664,9 @@ fn music_search_tracks(rp: RustyPipe, unlocalized: bool) { .items .iter() .find(|a| a.id == "BL-aIpCLWnU") - .unwrap(); + .unwrap_or_else(|| { + panic!("could not find track, got {:#?}", &res.items.items); + }); assert_eq!(track.name, "Black Mamba"); assert!(!track.cover.is_empty(), "got no cover"); @@ -1699,7 +1701,9 @@ fn music_search_videos(rp: RustyPipe, unlocalized: bool) { .items .iter() .find(|a| a.id == "ZeerrnuLi5E") - .unwrap(); + .unwrap_or_else(|| { + panic!("could not find video, got {:#?}", &res.items.items); + }); assert_eq!(track.name, "Black Mamba"); assert!(!track.cover.is_empty(), "got no cover"); @@ -1739,7 +1743,12 @@ fn music_search_episode(rp: RustyPipe, #[case] videos: bool) { .tracks }; - let track = &tracks.iter().find(|a| a.id == "Zq_-LDy7AgE").unwrap(); + let track = &tracks + .iter() + .find(|a| a.id == "Zq_-LDy7AgE") + .unwrap_or_else(|| { + panic!("could not find episode, got {:#?}", &tracks); + }); assert_eq!(track.artists.len(), 1); let track_artist = &track.artists[0]; @@ -1805,7 +1814,14 @@ fn music_search_albums( ) { let res = tokio_test::block_on(rp.query().music_search_albums(query)).unwrap(); - let album = &res.items.items.iter().find(|a| a.id == id).unwrap(); + let album = &res + .items + .items + .iter() + .find(|a| a.id == id) + .unwrap_or_else(|| { + panic!("could not find album, got {:#?}", &res.items.items); + }); assert_eq!(album.name, name); assert_eq!(album.artists.len(), 1); @@ -1836,7 +1852,9 @@ fn music_search_artists(rp: RustyPipe, unlocalized: bool) { .items .iter() .find(|a| a.id == "UCIh4j8fXWf2U0ro0qnGU8Mg") - .unwrap(); + .unwrap_or_else(|| { + panic!("could not find artist, got {:#?}", &res.items.items); + }); if unlocalized { assert_eq!(artist.name, "Namika"); } @@ -1871,7 +1889,9 @@ fn music_search_playlists(rp: RustyPipe, unlocalized: bool) { .items .iter() .find(|p| p.id == "RDCLAK5uy_nLtxizvEMkzYQUrA-bFf6MnBeR4bGYWUQ") - .expect("no playlist"); + .unwrap_or_else(|| { + panic!("could not find playlist, got {:#?}", &res.items.items); + }); if unlocalized { assert_eq!(playlist.name, "Today's Rock Hits"); @@ -1901,7 +1921,9 @@ fn music_search_playlists_community(rp: RustyPipe) { .items .iter() .find(|p| p.id == "PLMC9KNkIncKtGvr2kFRuXBVmBev6cAJ2u") - .expect("no playlist"); + .unwrap_or_else(|| { + panic!("could not find playlist, got {:#?}", &res.items.items); + }); assert_eq!( playlist.name,