feat: add music_related

README.md

@@ -25,7 +25,7 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
 - [X] **Search**
 - [ ] **Search suggestions**
 - [X] **Radio**
-- [ ] **Track details** (lyrics, recommendations)
+- [X] **Track details** (lyrics, recommendations)
 - [ ] **Moods**
 - [ ] **Charts**
 - [ ] **New**

codegen/src/download_testfiles.rs

@@ -51,6 +51,7 @@ pub async fn download_testfiles(project_root: &Path) {
     music_artist(&testfiles).await;
     music_details(&testfiles).await;
     music_lyrics(&testfiles).await;
+    music_related(&testfiles).await;
     music_radio(&testfiles).await;
     music_radio_cont(&testfiles).await;
 }
@@ -735,6 +736,24 @@ async fn music_lyrics(testfiles: &Path) {
         .unwrap();
 }
 
+async fn music_related(testfiles: &Path) {
+    let mut json_path = testfiles.to_path_buf();
+    json_path.push("music_details");
+    json_path.push("related.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = RustyPipe::new();
+    let res = rp.query().music_details("ZeerrnuLi5E").await.unwrap();
+
+    let rp = rp_testfile(&json_path);
+    rp.query()
+        .music_related(&res.related_id.unwrap())
+        .await
+        .unwrap();
+}
+
 async fn music_radio(testfiles: &Path) {
     for (name, id) in [("mv", "RDAMVMZeerrnuLi5E"), ("track", "RDAMVM7nigXQS1Xb0")] {
         let mut json_path = testfiles.to_path_buf();

src/client/music_artist.rs

@@ -193,25 +193,26 @@ fn map_artist_page(
             response::music_item::ItemSection::MusicCarouselShelfRenderer { header, contents } => {
                 let mut extendable_albums = false;
                 if let Some(h) = header {
-                    let ep = h
+                    if let Some(button) = h
                         .music_carousel_shelf_basic_header_renderer
                         .more_content_button
-                        .button_renderer
-                        .navigation_endpoint;
-
-                    if let Some(bep) = ep.browse_endpoint {
-                        if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
-                            match cfg.browse_endpoint_context_music_config.page_type {
-                                PageType::Playlist => {
-                                    if videos_playlist_id.is_none() {
-                                        videos_playlist_id = Some(bep.browse_id);
+                    {
+                        if let Some(bep) =
+                            button.button_renderer.navigation_endpoint.browse_endpoint
+                        {
+                            if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
+                                match cfg.browse_endpoint_context_music_config.page_type {
+                                    PageType::Playlist => {
+                                        if videos_playlist_id.is_none() {
+                                            videos_playlist_id = Some(bep.browse_id);
+                                        }
                                     }
+                                    PageType::Artist => {
+                                        album_page_params.push(bep.params);
+                                        extendable_albums = true;
+                                    }
+                                    _ => {}
                                 }
-                                PageType::Artist => {
-                                    album_page_params.push(bep.params);
-                                    extendable_albums = true;
-                                }
-                                _ => {}
                             }
                         }
                     }

src/client/music_details.rs

@@ -4,13 +4,16 @@ use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{Lyrics, Paginator, TrackDetails, TrackItem},
+    model::{ArtistId, Lyrics, MusicRelated, Paginator, TrackDetails, TrackItem},
     param::Language,
     serializer::MapResult,
 };
 
 use super::{
-    response::{self, music_item::map_queue_item},
+    response::{
+        self,
+        music_item::{map_queue_item, MusicListMapper},
+    },
     ClientType, MapResponse, QBrowse, RustyPipeQuery, YTContext,
 };
 
@@ -71,6 +74,23 @@ impl RustyPipeQuery {
         .await
     }
 
+    pub async fn music_related(&self, related_id: &str) -> Result<MusicRelated, Error> {
+        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
+        let request_body = QBrowse {
+            context,
+            browse_id: related_id,
+        };
+
+        self.execute_request::<response::MusicRelated, _, _>(
+            ClientType::DesktopMusic,
+            "music_related",
+            related_id,
+            "browse",
+            &request_body,
+        )
+        .await
+    }
+
     pub async fn music_radio(&self, radio_id: &str) -> Result<Paginator<TrackItem>, Error> {
         let context = self.get_context(ClientType::DesktopMusic, true, None).await;
         let request_body = QRadio {
@@ -256,6 +276,84 @@ impl MapResponse<Lyrics> for response::MusicLyrics {
     }
 }
 
+impl MapResponse<MusicRelated> for response::MusicRelated {
+    fn map_response(
+        self,
+        _id: &str,
+        lang: Language,
+        _deobf: Option<&crate::deobfuscate::Deobfuscator>,
+    ) -> Result<MapResult<MusicRelated>, ExtractionError> {
+        // Find artist
+        let artist_id = self
+            .contents
+            .section_list_renderer
+            .contents
+            .iter()
+            .find_map(|section| match section {
+                response::music_item::ItemSection::MusicShelfRenderer(_) => None,
+                response::music_item::ItemSection::MusicCarouselShelfRenderer {
+                    header, ..
+                } => header.as_ref().and_then(|h| {
+                    h.music_carousel_shelf_basic_header_renderer
+                        .title
+                        .0
+                        .iter()
+                        .find_map(|c| {
+                            let artist = ArtistId::from(c.clone());
+                            if artist.id.is_some() {
+                                Some(artist)
+                            } else {
+                                None
+                            }
+                        })
+                }),
+                response::music_item::ItemSection::None => None,
+            });
+
+        let mut mapper_tracks = MusicListMapper::new(lang);
+        let mut mapper = match artist_id {
+            Some(artist_id) => MusicListMapper::with_artist(lang, artist_id),
+            None => MusicListMapper::new(lang),
+        };
+
+        let mut sections = self.contents.section_list_renderer.contents.into_iter();
+        if let Some(response::music_item::ItemSection::MusicCarouselShelfRenderer {
+            contents,
+            ..
+        }) = sections.next()
+        {
+            mapper_tracks.map_response(contents);
+        }
+
+        sections.for_each(|section| match section {
+            response::music_item::ItemSection::MusicShelfRenderer(shelf) => {
+                mapper.map_response(shelf.contents);
+            }
+            response::music_item::ItemSection::MusicCarouselShelfRenderer { contents, .. } => {
+                mapper.map_response(contents);
+            }
+            response::music_item::ItemSection::None => {}
+        });
+
+        let mapped_tracks = mapper_tracks.conv_items();
+        let mut mapped = mapper.group_items();
+
+        let mut warnings = mapped_tracks.warnings;
+        warnings.append(&mut mapped.warnings);
+
+        Ok(MapResult {
+            c: MusicRelated {
+                tracks: mapped_tracks.c,
+                other_versions: mapped.c.tracks,
+                albums: mapped.c.albums,
+                artists: mapped.c.artists,
+                playlists: mapped.c.playlists,
+            },
+            warnings,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::{fs::File, io::BufReader, path::Path};
@@ -323,4 +421,21 @@ mod tests {
         );
         insta::assert_ron_snapshot!(format!("map_music_lyrics"), map_res.c);
     }
+
+    #[test]
+    fn map_related() {
+        let json_path = Path::new("testfiles/music_details/related.json");
+        let json_file = File::open(json_path).unwrap();
+
+        let lyrics: response::MusicRelated =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let map_res: MapResult<MusicRelated> = lyrics.map_response("", Language::En, None).unwrap();
+
+        assert!(
+            map_res.warnings.is_empty(),
+            "deserialization/mapping warnings: {:?}",
+            map_res.warnings
+        );
+        insta::assert_ron_snapshot!(format!("map_music_related"), map_res.c);
+    }
 }

src/client/response/mod.rs

@@ -17,6 +17,7 @@ pub(crate) use music_artist::MusicArtist;
 pub(crate) use music_artist::MusicArtistAlbums;
 pub(crate) use music_details::MusicDetails;
 pub(crate) use music_details::MusicLyrics;
+pub(crate) use music_details::MusicRelated;
 pub(crate) use music_item::MusicContinuation;
 pub(crate) use music_playlist::MusicPlaylist;
 pub(crate) use music_search::MusicSearch;

src/client/response/music_details.rs

@@ -3,7 +3,10 @@ use serde_with::serde_as;
 
 use crate::serializer::text::Text;
 
-use super::{music_item::PlaylistPanelRenderer, ContentRenderer, SectionList};
+use super::{
+    music_item::{ItemSection, PlaylistPanelRenderer},
+    ContentRenderer, SectionList,
+};
 
 /// Response model for YouTube Music track details
 #[derive(Debug, Deserialize)]
@@ -116,3 +119,9 @@ pub(crate) struct LyricsRenderer {
     #[serde_as(as = "Text")]
     pub footer: String,
 }
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct MusicRelated {
+    pub contents: SectionList<ItemSection>,
+}

src/client/response/music_item.rs

@@ -27,8 +27,6 @@ pub(crate) enum ItemSection {
     #[serde(alias = "musicPlaylistShelfRenderer")]
     MusicShelfRenderer(MusicShelf),
     MusicCarouselShelfRenderer {
-        #[serde(default)]
-        #[serde_as(as = "DefaultOnError")]
         header: Option<MusicCarouselShelfHeader>,
         #[serde_as(as = "VecLogError<_>")]
         contents: MapResult<Vec<MusicResponseItem>>,
@@ -136,13 +134,15 @@ pub(crate) struct ListMusicItem {
     pub navigation_endpoint: Option<NavigationEndpoint>,
     #[serde(default)]
     pub flex_column_display_style: FlexColumnDisplayStyle,
+    #[serde(default)]
+    pub item_height: ItemHeight,
     /// Album track number
     #[serde_as(as = "Option<Text>")]
     pub index: Option<String>,
     pub menu: Option<MusicItemMenu>,
 }
 
-#[derive(Default, Debug, Deserialize)]
+#[derive(Default, Debug, Copy, Clone, Deserialize)]
 pub(crate) enum FlexColumnDisplayStyle {
     #[serde(rename = "MUSIC_RESPONSIVE_LIST_ITEM_FLEX_COLUMN_DISPLAY_STYLE_TWO_LINE_STACK")]
     TwoLines,
@@ -151,6 +151,15 @@ pub(crate) enum FlexColumnDisplayStyle {
     Default,
 }
 
+#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Deserialize)]
+pub(crate) enum ItemHeight {
+    #[serde(rename = "MUSIC_RESPONSIVE_LIST_ITEM_HEIGHT_MEDIUM_COMPACT")]
+    Compact,
+    #[default]
+    #[serde(other)]
+    Default,
+}
+
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -295,7 +304,9 @@ pub(crate) struct MusicCarouselShelfHeader {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicCarouselShelfHeaderRenderer {
-    pub more_content_button: MoreContentButton,
+    pub more_content_button: Option<MoreContentButton>,
+    #[serde(default)]
+    pub title: TextComponents,
 }
 
 #[derive(Debug, Deserialize)]
@@ -523,27 +534,42 @@ impl MusicListMapper {
                         {
                             // Search result
                             FlexColumnDisplayStyle::TwoLines => {
-                                let mut subtitle_parts = c2
-                                    .ok_or_else(|| format!("track {}: could not get subtitle", id))?
-                                    .renderer
-                                    .text
-                                    .split(util::DOT_SEPARATOR)
-                                    .into_iter();
-
-                                // Is it a podcast episode?
-                                if subtitle_parts.len() <= 3 && c3.is_some() {
-                                    (subtitle_parts.rev().next(), None, None)
-                                } else {
-                                    // Skip first part (track type)
-                                    if subtitle_parts.len() > 3 {
-                                        subtitle_parts.next();
-                                    }
-
+                                // Is this a related track?
+                                if !is_video && item.item_height == ItemHeight::Compact {
                                     (
-                                        subtitle_parts.next(),
-                                        subtitle_parts.next(),
-                                        subtitle_parts.next(),
+                                        c2.map(TextComponents::from),
+                                        c3.map(TextComponents::from),
+                                        None,
                                     )
+                                } else {
+                                    let mut subtitle_parts = c2
+                                        .ok_or_else(|| {
+                                            format!("track {}: could not get subtitle", id)
+                                        })?
+                                        .renderer
+                                        .text
+                                        .split(util::DOT_SEPARATOR)
+                                        .into_iter();
+
+                                    // Is this a related video?
+                                    if item.item_height == ItemHeight::Compact {
+                                        (subtitle_parts.next(), subtitle_parts.next(), None)
+                                    }
+                                    // Is it a podcast episode?
+                                    else if subtitle_parts.len() <= 3 && c3.is_some() {
+                                        (subtitle_parts.rev().next(), None, None)
+                                    } else {
+                                        // Skip first part (track type)
+                                        if subtitle_parts.len() > 3 {
+                                            subtitle_parts.next();
+                                        }
+
+                                        (
+                                            subtitle_parts.next(),
+                                            subtitle_parts.next(),
+                                            subtitle_parts.next(),
+                                        )
+                                    }
                                 }
                             }
                             // Playlist item

src/model/mod.rs

@@ -1240,9 +1240,28 @@ pub struct TrackDetails {
     pub related_id: Option<String>,
 }
 
+/// Song lyrics
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[non_exhaustive]
 pub struct Lyrics {
+    /// Lyrics text
     pub body: String,
+    /// Footer (contains lyrics source)
     pub footer: String,
 }
+
+/// YouTube Music related entities
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[non_exhaustive]
+pub struct MusicRelated {
+    /// Related tracks
+    pub tracks: Vec<TrackItem>,
+    /// Other versions of the same track
+    pub other_versions: Vec<TrackItem>,
+    /// Related albums
+    pub albums: Vec<AlbumItem>,
+    /// Related artists
+    pub artists: Vec<ArtistItem>,
+    /// Related playlists
+    pub playlists: Vec<MusicPlaylistItem>,
+}

tests/youtube.rs

@@ -1,6 +1,8 @@
 use std::collections::HashSet;
 use std::fmt::Display;
 
+use fancy_regex::Regex;
+use once_cell::sync::Lazy;
 use rstest::rstest;
 use time::macros::date;
 use time::OffsetDateTime;
@@ -1781,6 +1783,97 @@ async fn music_lyrics() {
     insta::assert_ron_snapshot!(lyrics);
 }
 
+#[rstest]
+#[case::a("7nigXQS1Xb0", true)]
+#[case::b("4t3SUDZCBaQ", false)]
+#[tokio::test]
+async fn music_related(#[case] id: &str, #[case] full: bool) {
+    let rp = RustyPipe::builder().strict().build();
+    let track = rp.query().music_details(id).await.unwrap();
+    let related = rp
+        .query()
+        .music_related(&track.related_id.unwrap())
+        .await
+        .unwrap();
+
+    let n_tracks = related.tracks.len();
+    let mut track_artists = 0;
+    let mut track_artist_ids = 0;
+    let mut n_tracks_ytm = 0;
+    let mut track_albums = 0;
+
+    for track in related.tracks {
+        assert_video_id(&track.id);
+        assert!(!track.title.is_empty());
+        assert!(!track.cover.is_empty(), "got no cover");
+
+        if let Some(artist_id) = track.artist_id {
+            assert_channel_id(&artist_id);
+            track_artist_ids += 1;
+        }
+
+        let artist = track.artists.first().unwrap();
+        assert!(!artist.name.is_empty());
+        if let Some(artist_id) = &artist.id {
+            assert_channel_id(artist_id);
+            track_artists += 1;
+        }
+
+        if track.is_video {
+            assert!(track.album.is_none());
+            assert_gte(track.view_count.unwrap(), 10_000, "views")
+        } else {
+            n_tracks_ytm += 1;
+
+            assert!(track.view_count.is_none());
+            if let Some(album) = track.album {
+                assert_album_id(&album.id);
+                assert!(!album.name.is_empty());
+                track_albums += 1;
+            }
+        }
+    }
+
+    assert_gte(n_tracks, 20, "tracks");
+    assert_gte(n_tracks_ytm, 10, "tracks_ytm");
+
+    assert_gte(track_artists, n_tracks - 3, "track_artists");
+    assert_gte(track_artist_ids, n_tracks - 3, "track_artists");
+    assert_gte(track_albums, n_tracks_ytm - 3, "track_artists");
+
+    if full {
+        assert_gte(related.albums.len(), 10, "albums");
+        for album in related.albums {
+            assert_album_id(&album.id);
+            assert!(!album.name.is_empty());
+            assert!(!album.cover.is_empty(), "got no cover");
+
+            let artist = album.artists.first().unwrap();
+            assert_channel_id(&artist.id.as_ref().unwrap());
+            assert!(!artist.name.is_empty());
+        }
+
+        assert_gte(related.artists.len(), 10, "artists");
+        for artist in related.artists {
+            assert_channel_id(&artist.id);
+            assert!(!artist.name.is_empty());
+            assert!(!artist.avatar.is_empty(), "got no avatar");
+            assert_gte(artist.subscriber_count.unwrap(), 5000, "subscribers")
+        }
+
+        assert_gte(related.playlists.len(), 10, "playlists");
+        for playlist in related.playlists {
+            assert_playlist_id(&playlist.id);
+            assert!(!playlist.name.is_empty());
+            assert!(!playlist.thumbnail.is_empty(), "got no playlist thumbnail");
+            let channel = playlist.channel.unwrap();
+            assert_channel_id(&channel.id);
+            assert!(!channel.name.is_empty());
+            assert_gte(playlist.track_count.unwrap(), 2, "tracks");
+        }
+    }
+}
+
 #[tokio::test]
 async fn music_radio_track() {
     let rp = RustyPipe::builder().strict().build();
@@ -1836,3 +1929,46 @@ async fn assert_next<T: FromYtItem>(
         );
     }
 }
+
+fn assert_video_id(id: &str) {
+    static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-]{11}$").unwrap());
+
+    assert!(
+        VIDEO_ID_REGEX.is_match(id).unwrap_or_default(),
+        "invalid video id: `{}`",
+        id
+    );
+}
+
+fn assert_channel_id(id: &str) {
+    static CHANNEL_ID_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap());
+
+    assert!(
+        CHANNEL_ID_REGEX.is_match(id).unwrap_or_default(),
+        "invalid channel id: `{}`",
+        id
+    );
+}
+
+fn assert_album_id(id: &str) {
+    static ALBUM_ID_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap());
+
+    assert!(
+        ALBUM_ID_REGEX.is_match(id).unwrap_or_default(),
+        "invalid album id: `{}`",
+        id
+    );
+}
+
+fn assert_playlist_id(id: &str) {
+    static PLAYLIST_ID_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK)[A-Za-z0-9_-]{30,}$").unwrap());
+
+    assert!(
+        PLAYLIST_ID_REGEX.is_match(id).unwrap_or_default(),
+        "invalid album id: `{}`",
+        id
+    );
+}