feat: add playlist extraction

- replace original base.js with dummy

Cargo.toml

@@ -37,4 +37,3 @@ rstest = "0.15.0"
 temp_testdir = "0.2.3"
 insta = "1.17.1"
 velcro = "0.5.3"
- 

cli/src/main.rs

@@ -148,12 +148,12 @@ async fn download_playlist(
         .expect("unable to build the HTTP client");
 
     let rt = RustyTube::new();
-    let playlist = rt.get_playlist(id, ClientType::Desktop).await.unwrap();
+    let playlist = rt.get_playlist(id).await.unwrap();
 
     // Indicatif setup
     let multi = MultiProgress::new();
     let main = multi.add(ProgressBar::new(
-        playlist.len().try_into().unwrap_or_default(),
+        playlist.videos.len().try_into().unwrap_or_default(),
     ));
 
     main.set_style(
@@ -164,11 +164,11 @@ async fn download_playlist(
     );
     main.tick();
 
-    stream::iter(playlist)
+    stream::iter(playlist.videos)
-        .map(|item| {
+        .map(|video| {
             download_single_video(
-                item.video_id.to_owned(),
+                video.id.to_owned(),
-                item.title.to_owned(),
+                video.title.to_owned(),
                 output_dir,
                 output_fname.to_owned(),
                 resolution,

notes/video_ids.txt

@@ -19,3 +19,10 @@ DRM: 1bfOsni7EgI
 Album with unknown artists: https://music.youtube.com/playlist?list=OLAK5uy_mEX9ljZeeEWgTM1xLL1isyiGaWXoPyoOk
 
 Throttling issue: Y8JFxS1HlDo
+
+# Playlists
+962 Songs: PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI
+97 Songs, YTM: RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk
+495 Songs: PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ
+78 Videos: PL2_OBreMn7FpDFj9lWfoZ8OQJvZkQa3yG
+66 Videos: PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe

src/client/player.rs

@@ -52,7 +52,7 @@ struct QContentPlaybackContext {
 }
 
 impl RustyTube {
-    pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<PlayerData> {
+    pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
         let client = self.get_ytclient(client_type);
         let (context, deobf) = tokio::join!(
             client.get_context(false),
@@ -315,7 +315,7 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec {
     AudioCodec::Unknown
 }
 
-fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<PlayerData> {
+fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<VideoPlayer> {
     // Check playability status
     match response.playability_status {
         response::player::PlayabilityStatus::Ok { live_streamability } => {
@@ -363,10 +363,10 @@ fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<P
                 width: t.width,
             })
             .collect(),
-
+        channel: Channel {
-        channel_id: video_details.channel_id,
+            id: video_details.channel_id,
-        channel_name: video_details.author,
+            name: video_details.author,
-
+        },
         publish_date: microformat.as_ref().map(|m| {
             let ndt = NaiveDateTime::new(m.publish_date, NaiveTime::from_hms(0, 0, 0));
             DateTime::from_utc(ndt, Utc)
@@ -434,7 +434,7 @@ fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<P
             .collect()
     });
 
-    Ok(PlayerData {
+    Ok(VideoPlayer {
         info: video_info,
         video_streams,
         video_only_streams,
@@ -446,7 +446,7 @@ fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<P
 
 #[cfg(test)]
 mod tests {
-    use std::path::Path;
+    use std::{fs::File, io::BufReader, path::Path};
 
     use crate::{cache::DeobfData, client::CLIENT_TYPES};
 
@@ -491,7 +491,7 @@ mod tests {
                 .error_for_status()
                 .unwrap();
 
-            let mut file = std::fs::File::create(json_path).unwrap();
+            let mut file = File::create(json_path).unwrap();
             let mut content = std::io::Cursor::new(resp.bytes().await.unwrap());
             std::io::copy(&mut content, &mut file).unwrap();
         }
@@ -510,23 +510,40 @@ mod tests {
             }
 
             let player_data = rt.get_player(id, ClientType::Desktop).await.unwrap();
-            let file = std::fs::File::create(json_path).unwrap();
+            let file = File::create(json_path).unwrap();
             serde_json::to_writer_pretty(file, &player_data).unwrap();
         }
     }
 
     #[rstest]
-    #[case::desktop("desktop", include_str!("../../testfiles/player/desktop_video.json"))]
+    #[case::desktop("desktop")]
-    #[case::desktop_music("desktop_music", include_str!("../../testfiles/player/desktopmusic_video.json"))]
+    #[case::desktop_music("desktopmusic")]
-    #[case::tv_html5_embed("tvhtml5embed", include_str!("../../testfiles/player/tvhtml5embed_video.json"))]
+    #[case::tv_html5_embed("tvhtml5embed")]
-    #[case::android("android", include_str!("../../testfiles/player/android_video.json"))]
+    #[case::android("android")]
-    #[case::ios("ios", include_str!("../../testfiles/player/ios_video.json"))]
+    #[case::ios("ios")]
-    fn t_map_player_data(#[case] name: &str, #[case] json_str: &str) {
+    fn t_map_player_data(#[case] name: &str) {
-        let resp = serde_json::from_str::<response::Player>(json_str).unwrap();
+        let filename = format!("testfiles/player/{}_video.json", name);
+        let json_path = Path::new(&filename);
+        let json_file = File::open(json_path).unwrap();
+
+        let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap();
         let player_data = map_player_data(resp, &DEOBFUSCATOR).unwrap();
         insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), player_data)
     }
 
+    /// Assert equality within 10% margin
+    fn assert_approx(left: u32, right: u32) {
+        if left != right {
+            let f = left as f64 / right as f64;
+            assert!(
+                0.9 < f && f < 1.1,
+                "{} not within 10% margin of {}",
+                left,
+                right
+            );
+        }
+    }
+
     #[rstest]
     #[case::desktop(ClientType::Desktop)]
     #[case::tv_html5_embed(ClientType::TvHtml5Embed)]
@@ -550,8 +567,8 @@ mod tests {
         }
         assert_eq!(player_data.info.length, 259);
         assert!(!player_data.info.thumbnails.is_empty());
-        assert_eq!(player_data.info.channel_id, "UC_aEa8K-EOJ3D6gOs7HcyNg");
+        assert_eq!(player_data.info.channel.id, "UC_aEa8K-EOJ3D6gOs7HcyNg");
-        assert_eq!(player_data.info.channel_name, "NoCopyrightSounds");
+        assert_eq!(player_data.info.channel.name, "NoCopyrightSounds");
         assert!(player_data.info.view_count > 146818808);
         assert_eq!(player_data.info.keywords[0], "spektrem");
         assert_eq!(player_data.info.is_live_content, false);
@@ -577,7 +594,8 @@ mod tests {
                 .find(|s| s.itag == 140)
                 .unwrap();
 
-            assert_eq!(video.bitrate, 1507068);
+            // Bitrates may change between requests
+            assert_approx(video.bitrate, 1507068);
             assert_eq!(video.average_bitrate, 1345149);
             assert_eq!(video.size, 43553412);
             assert_eq!(video.width, 1280);
@@ -589,7 +607,7 @@ mod tests {
             assert_eq!(video.format, VideoFormat::Webm);
             assert_eq!(video.codec, VideoCodec::Vp9);
 
-            assert_eq!(audio.bitrate, 130685);
+            assert_approx(audio.bitrate, 130685);
             assert_eq!(audio.average_bitrate, 129496);
             assert_eq!(audio.size, 4193863);
             assert_eq!(audio.mime, "audio/mp4; codecs=\"mp4a.40.2\"");
@@ -607,7 +625,7 @@ mod tests {
                 .find(|s| s.itag == 251)
                 .unwrap();
 
-            assert_eq!(video.bitrate, 1340829);
+            assert_approx(video.bitrate, 1340829);
             assert_eq!(video.average_bitrate, 1233444);
             assert_eq!(video.size, 39936630);
             assert_eq!(video.width, 1280);
@@ -620,7 +638,7 @@ mod tests {
             assert_eq!(video.codec, VideoCodec::Av01);
             assert_eq!(video.throttled, false);
 
-            assert_eq!(audio.bitrate, 142718);
+            assert_approx(audio.bitrate, 142718);
             assert_eq!(audio.average_bitrate, 130708);
             assert_eq!(audio.size, 4232344);
             assert_eq!(audio.mime, "audio/webm; codecs=\"opus\"");

src/client/playlist.rs

@@ -1,9 +1,14 @@
 // REQUEST
 
-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, Result};
 use reqwest::Method;
 use serde::Serialize;
 
+use crate::{
+    model::{Channel, Playlist, Thumbnail, Video},
+    serializer::text::{PageType, Text, TextLink},
+};
+
 use super::{response, ClientType, ContextYT, RustyTube};
 
 #[derive(Clone, Debug, Serialize)]
@@ -20,13 +25,8 @@ pub struct TmpEntry {
 }
 
 impl RustyTube {
-    pub async fn get_playlist(
+    pub async fn get_playlist(&self, playlist_id: &str) -> Result<Playlist> {
-        &self,
+        let client = self.get_ytclient(ClientType::Desktop);
-        playlist_id: &str,
-        client_type: ClientType,
-    ) -> Result<Vec<TmpEntry>> {
-        // let client = self.desktop_client.clone();
-        let client = self.get_ytclient(client_type);
         let context = client.get_context(true).await;
 
         let request_body = QPlaylist {
@@ -44,57 +44,204 @@ impl RustyTube {
 
         let playlist_response = resp.json::<response::Playlist>().await?;
 
-        Ok(map_playlist_tmp(playlist_response))
+        map_playlist(&playlist_response)
     }
 }
 
-fn map_playlist_tmp(response: response::Playlist) -> Vec<TmpEntry> {
+fn map_playlist(response: &response::Playlist) -> Result<Playlist> {
-    let content = &response
+    let video_items = &some_or_bail!(
-        .contents
+        some_or_bail!(
-        .two_column_browse_results_renderer
+            some_or_bail!(
-        .contents[0]
+                response
-        .tab_renderer
+                    .contents
-        .content
+                    .two_column_browse_results_renderer
-        .section_list_renderer
+                    .contents
-        .contents[0];
+                    .get(0),
-
+                Err(anyhow!("twoColumnBrowseResultsRenderer empty"))
-    match &content.item_section_renderer {
+            )
-        Some(items) => items.contents[0]
+            .tab_renderer
-            .playlist_video_list_renderer
+            .content
+            .section_list_renderer
             .contents
+            .get(0),
+            Err(anyhow!("sectionListRenderer empty"))
+        )
+        .item_section_renderer
+        .contents
+        .get(0),
+        Err(anyhow!("itemSectionRenderer empty"))
+    )
+    .playlist_video_list_renderer
+    .contents;
+
+    let mut ctoken: Option<String> = None;
+    let videos = video_items
+        .iter()
+        .filter_map(|it| match it {
+            response::playlist::PlaylistVideoItem::PlaylistVideoRenderer { video } => {
+                match &video.channel {
+                    TextLink::Browse {
+                        text,
+                        page_type,
+                        browse_id,
+                    } => match page_type {
+                        PageType::Channel => Some(Video {
+                            id: video.video_id.to_owned(),
+                            title: video.title.to_owned(),
+                            length: video.length_seconds,
+                            thumbnails: video
+                                .thumbnail
+                                .thumbnails
+                                .iter()
+                                .map(|t| Thumbnail {
+                                    url: t.url.to_owned(),
+                                    width: t.width,
+                                    height: t.height,
+                                })
+                                .collect(),
+                            channel: Channel {
+                                id: browse_id.to_string(),
+                                name: text.to_owned(),
+                            },
+                        }),
+                        _ => None,
+                    },
+                    _ => None,
+                }
+            }
+            response::playlist::PlaylistVideoItem::ContinuationItemRenderer {
+                continuation_endpoint,
+            } => {
+                ctoken = Some(continuation_endpoint.continuation_command.token.to_owned());
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let thumbnail_renderer = some_or_bail!(
+        response
+            .sidebar
+            .playlist_sidebar_renderer
+            .items
             .iter()
-            .map(|it| TmpEntry {
+            .find_map(|s| match s {
-                title: it.playlist_video_renderer.title.to_owned(),
+                response::playlist::SidebarRendererItem::PlaylistSidebarPrimaryInfoRenderer {
-                video_id: it.playlist_video_renderer.video_id.to_owned(),
+                    thumbnail_renderer,
-            })
+                } => Some(thumbnail_renderer),
-            .collect(),
+                _ => None,
-        None => todo!(),
+            }),
-    }
+        Err(anyhow!("no primary sidebar"))
+    );
+
+    let video_owner_wrap = response
+        .sidebar
+        .playlist_sidebar_renderer
+        .items
+        .iter()
+        .find_map(|s| match s {
+            response::playlist::SidebarRendererItem::PlaylistSidebarSecondaryInfoRenderer {
+                video_owner,
+            } => Some(video_owner),
+            _ => None,
+        });
+
+    let n_videos = match ctoken {
+        Some(_) => {
+            some_or_bail!(
+                match &response.header.playlist_header_renderer.num_videos_text {
+                    Text::Multiple { runs } =>
+                        if runs.len() == 2 && runs[1] == " videos" {
+                            runs[0].parse().ok()
+                        } else {
+                            None
+                        },
+                    _ => None,
+                },
+                Err(anyhow!("no video count"))
+            )
+        }
+        None => videos.len() as u32,
+    };
+
+    let thumbnails = thumbnail_renderer
+        .playlist_video_thumbnail_renderer
+        .thumbnail
+        .thumbnails
+        .iter()
+        .map(|t| Thumbnail {
+            url: t.url.to_owned(),
+            width: t.width,
+            height: t.height,
+        })
+        .collect::<Vec<_>>();
+
+    let name = response.header.playlist_header_renderer.title.to_owned();
+    let description = response
+        .header
+        .playlist_header_renderer
+        .description_text
+        .to_owned();
+
+    let channel = match video_owner_wrap {
+        Some(o) => match &o.video_owner_renderer.title {
+            TextLink::Browse {
+                text,
+                page_type,
+                browse_id,
+            } => match page_type {
+                PageType::Channel => Some(Channel {
+                    id: browse_id.to_owned(),
+                    name: text.to_owned(),
+                }),
+                _ => None,
+            },
+            _ => None,
+        },
+        None => None,
+    };
+
+    Ok(Playlist {
+        videos,
+        n_videos,
+        ctoken,
+        name,
+        thumbnails,
+        description,
+        channel,
+    })
 }
 
 #[cfg(test)]
 mod tests {
-    use std::path::Path;
+    use std::{fs::File, io::BufReader, path::Path};
 
-    use crate::client::ClientType;
+    use rstest::rstest;
 
     use super::*;
 
-    #[allow(dead_code)]
+    #[test_log::test(tokio::test)]
-    // #[test_log::test(tokio::test)]
     async fn download_testfiles() {
         let tf_dir = Path::new("testfiles/playlist");
-        let playlist_id = "RDCLAK5uy_mHW5bcduhjB-PkTePAe6EoRMj1xNT8gzY";
 
         let rt = RustyTube::new();
 
-        for client_type in [ClientType::Desktop, ClientType::DesktopMusic] {
+        for (name, id) in [
-            let client = rt.get_ytclient(client_type);
+            ("short", "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk"),
+            ("long", "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ"),
+            ("nomusic", "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe"),
+        ] {
+            let mut json_path = tf_dir.to_path_buf();
+            json_path.push(format!("playlist_{}.json", name));
+            if json_path.exists() {
+                continue;
+            }
+
+            let client = rt.get_ytclient(ClientType::Desktop);
             let context = client.get_context(false).await;
 
             let request_body = QPlaylist {
                 context,
-                browse_id: "VL".to_owned() + playlist_id,
+                browse_id: "VL".to_owned() + id,
             };
 
             let resp = client
@@ -107,40 +254,73 @@ mod tests {
                 .error_for_status()
                 .unwrap();
 
-            let mut json_path = tf_dir.to_path_buf();
-            json_path.push(format!("{:?}_playlist.json", client_type).to_lowercase());
-
             let mut file = std::fs::File::create(json_path).unwrap();
             let mut content = std::io::Cursor::new(resp.bytes().await.unwrap());
             std::io::copy(&mut content, &mut file).unwrap();
         }
     }
 
+    #[rstest]
+    #[case::long(
+        "PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ",
+        "Die schönsten deutschen Lieder | Beliebteste Lieder | Beste Deutsche Musik 2022",
+        true,
+        None,
+        Some(Channel {
+            id: "UCIekuFeMaV78xYfvpmoCnPg".to_owned(),
+            name: "Best Music".to_owned(),
+        })
+    )]
+    #[case::short(
+        "RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk",
+        "Easy Pop",
+        false,
+        None,
+        None
+    )]
+    #[case::nomusic(
+        "PL1J-6JOckZtE_P9Xx8D3b2O6w0idhuKBe",
+        "Minecraft SHINE",
+        false,
+        Some("SHINE - Survival Hardcore in New Environment: Auf einem Server machen sich tapfere Spieler auf, mystische Welten zu erkunden, magische Technologien zu erforschen und vorallem zu überleben...".to_owned()),
+        Some(Channel {
+            id: "UCQM0bS4_04-Y4JuYrgmnpZQ".to_owned(),
+            name: "Chaosflo44".to_owned(),
+        })
+    )]
     #[test_log::test(tokio::test)]
-    async fn t_get_playlist() {
+    async fn t_get_playlist(
+        #[case] id: &str,
+        #[case] name: &str,
+        #[case] is_long: bool,
+        #[case] description: Option<String>,
+        #[case] channel: Option<Channel>,
+    ) {
         let rt = RustyTube::new();
-        let playlist = rt
+        let playlist = rt.get_playlist(id).await.unwrap();
-            .get_playlist(
-                "RDCLAK5uy_mHW5bcduhjB-PkTePAe6EoRMj1xNT8gzY",
-                ClientType::Desktop,
-            )
-            .await
-            .unwrap();
 
-        dbg!(playlist);
+        assert_eq!(playlist.name, name);
+        assert!(!playlist.videos.is_empty());
+        assert_eq!(playlist.ctoken.is_some(), is_long);
+        assert!(playlist.n_videos > 10);
+        assert_eq!(playlist.n_videos > 100, is_long);
+        assert_eq!(playlist.description, description);
+        assert_eq!(playlist.channel, channel);
+        assert!(!playlist.thumbnails.is_empty());
     }
 
-    #[test_log::test(tokio::test)]
+    #[rstest]
-    async fn t_get_playlist_music() {
+    #[case::long("long")]
-        let rt = RustyTube::new();
+    #[case::short("short")]
-        let playlist = rt
+    #[case::nomusic("nomusic")]
-            .get_playlist(
+    fn t_map_player_data(#[case] name: &str) {
-                "RDCLAK5uy_mHW5bcduhjB-PkTePAe6EoRMj1xNT8gzY",
+        let filename = format!("testfiles/playlist/playlist_{}.json", name);
-                ClientType::Desktop,
+        let json_path = Path::new(&filename);
-            )
+        let json_file = File::open(json_path).unwrap();
-            .await
-            .unwrap();
 
-        dbg!(playlist);
+        let playlist: response::Playlist =
+            serde_json::from_reader(BufReader::new(json_file)).unwrap();
+        let playlist_data = map_playlist(&playlist).unwrap();
+        insta::assert_yaml_snapshot!(format!("map_playlist_data_{}", name), playlist_data);
     }
 }

src/client/response/mod.rs

@@ -1,14 +1,35 @@
 pub mod player;
 pub mod playlist;
+pub mod playlist_music;
 
 pub use player::Player;
 pub use playlist::Playlist;
+pub use playlist_music::PlaylistMusic;
 
 use serde::Deserialize;
-use serde_with::{serde_as, VecSkipError};
+use serde_with::{serde_as, DefaultOnError, VecSkipError};
 
 use crate::serializer::text::TextLink;
 
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContentRenderer<T> {
+    pub content: T,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContentsRenderer<T> {
+    #[serde(alias = "tabs")]
+    pub contents: Vec<T>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ThumbnailsWrap {
+    pub thumbnail: Thumbnails,
+}
+
 #[derive(Default, Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Thumbnails {
@@ -23,6 +44,24 @@ pub struct Thumbnail {
     pub height: u32,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationItemRenderer {
+    pub continuation_endpoint: ContinuationEndpoint,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationEndpoint {
+    pub continuation_command: ContinuationCommand,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContinuationCommand {
+    pub token: String,
+}
+
 // YouTube Music
 
 #[serde_as]
@@ -30,7 +69,9 @@ pub struct Thumbnail {
 #[serde(rename_all = "camelCase")]
 pub struct MusicItem {
     pub thumbnail: MusicThumbnailRenderer,
-    pub playlist_item_data: PlaylistItemData,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub playlist_item_data: Option<PlaylistItemData>,
     #[serde(default)]
     #[serde_as(as = "VecSkipError<_>")]
     pub flex_columns: Vec<MusicColumn>,
@@ -42,13 +83,8 @@ pub struct MusicItem {
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct MusicThumbnailRenderer {
-    pub music_thumbnail_renderer: MusicThumbnailRenderer2,
+    #[serde(alias = "croppedSquareThumbnailRenderer")]
-}
+    pub music_thumbnail_renderer: ThumbnailsWrap,
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct MusicThumbnailRenderer2 {
-    pub thumbnail: Thumbnails,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -57,6 +93,15 @@ pub struct PlaylistItemData {
     pub video_id: String,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicContentsRenderer<T> {
+    pub contents: Vec<T>,
+    #[serde_as(as = "Option<VecSkipError<_>>")]
+    pub continuations: Option<Vec<MusicContinuation>>,
+}
+
 #[derive(Clone, Debug, Deserialize)]
 pub struct MusicColumn {
     #[serde(
@@ -69,6 +114,18 @@ pub struct MusicColumn {
 #[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 pub struct MusicColumnRenderer {
-    #[serde_as(as = "crate::serializer::text::TextLink")]
+    #[serde_as(as = "crate::serializer::text::TextLinks")]
-    pub text: TextLink,
+    pub text: Vec<TextLink>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicContinuation {
+    pub next_continuation_data: MusicContinuationData,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicContinuationData {
+    pub continuation: String,
 }

src/client/response/playlist.rs

@@ -2,21 +2,21 @@ use serde::Deserialize;
 use serde_with::serde_as;
 use serde_with::{json::JsonString, DefaultOnError, VecSkipError};
 
-use crate::serializer::text::TextLink;
+use crate::serializer::text::{Text, TextLink};
 
-use super::{MusicItem, Thumbnails};
+use super::{ContentRenderer, ContentsRenderer, ContinuationEndpoint, Thumbnails, ThumbnailsWrap};
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Playlist {
     pub contents: Contents,
     pub header: Header,
+    pub sidebar: Sidebar,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Contents {
-    #[serde(alias = "singleColumnBrowseResultsRenderer")]
     pub two_column_browse_results_renderer: ContentsRenderer<Tab>,
 }
 
@@ -35,26 +35,35 @@ pub struct SectionList {
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct ItemSection {
-    pub item_section_renderer: Option<ContentsRenderer<PlaylistVideoList>>,
+    pub item_section_renderer: ContentsRenderer<PlaylistVideoListRenderer>,
-    pub music_playlist_shelf_renderer: Option<ContentsRenderer<PlaylistMusicItem>>,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PlaylistVideoListRenderer {
+    pub playlist_video_list_renderer: PlaylistVideoList,
+}
+
+#[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct PlaylistVideoList {
-    pub playlist_video_list_renderer: ContentsRenderer<PlaylistVideoItem>,
+    #[serde_as(as = "VecSkipError<_>")]
+    pub contents: Vec<PlaylistVideoItem>,
 }
 
+#[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct PlaylistVideoItem {
+pub enum PlaylistVideoItem {
-    pub playlist_video_renderer: PlaylistVideo,
+    PlaylistVideoRenderer {
-}
+        #[serde(flatten)]
-
+        video: PlaylistVideo,
-#[derive(Clone, Debug, Deserialize)]
+    },
-#[serde(rename_all = "camelCase")]
+    #[serde(rename_all = "camelCase")]
-pub struct PlaylistMusicItem {
+    ContinuationItemRenderer {
-    pub music_responsive_list_item_renderer: MusicItem,
+        continuation_endpoint: ContinuationEndpoint,
+    },
 }
 
 #[serde_as]
@@ -76,7 +85,6 @@ pub struct PlaylistVideo {
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Header {
-    #[serde(alias = "musicDetailHeaderRenderer")]
     pub playlist_header_renderer: HeaderRenderer,
 }
 
@@ -84,22 +92,63 @@ pub struct Header {
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct HeaderRenderer {
-    pub playlist_id: Option<String>,
+    pub playlist_id: String,
     #[serde_as(as = "crate::serializer::text::Text")]
     pub title: String,
-    #[serde_as(as = "Option<crate::serializer::text::Text>")]
+    #[serde(default)]
-    pub description: Option<String>,
+    #[serde_as(as = "DefaultOnError<Option<crate::serializer::text::Text>>")]
+    pub description_text: Option<String>,
+    /// `"495", " videos"`
+    pub num_videos_text: Text,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct ContentRenderer<T> {
+pub struct Sidebar {
-    pub content: T,
+    pub playlist_sidebar_renderer: SidebarRenderer,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SidebarRenderer {
+    #[serde_as(as = "VecSkipError<_>")]
+    pub items: Vec<SidebarRendererItem>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub struct ContentsRenderer<T> {
+pub enum SidebarRendererItem {
-    #[serde(alias = "tabs")]
+    #[serde(rename_all = "camelCase")]
-    pub contents: Vec<T>,
+    PlaylistSidebarPrimaryInfoRenderer {
+        thumbnail_renderer: PlaylistThumbnailRenderer,
+        // - `"495", " videos"`
+        // - `"3,310,996 views"`
+        // - `"Last updated on ", "Aug 7, 2022"`
+        // stats: Vec<Text>,
+    },
+    #[serde(rename_all = "camelCase")]
+    PlaylistSidebarSecondaryInfoRenderer { video_owner: VideoOwnerWrap },
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PlaylistThumbnailRenderer {
+    // the alternative field name is used by YTM playlists
+    #[serde(alias = "playlistCustomThumbnailRenderer")]
+    pub playlist_video_thumbnail_renderer: ThumbnailsWrap,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VideoOwnerWrap {
+    pub video_owner_renderer: VideoOwner,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct VideoOwner {
+    #[serde_as(as = "crate::serializer::text::TextLink")]
+    pub title: TextLink,
 }

src/client/response/playlist_music.rs

@@ -0,0 +1,95 @@
+use serde::Deserialize;
+use serde_with::serde_as;
+use serde_with::VecSkipError;
+
+use crate::serializer::text::Text;
+
+use super::MusicThumbnailRenderer;
+use super::{
+    ContentRenderer, ContentsRenderer, MusicContentsRenderer, MusicContinuation, MusicItem,
+};
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PlaylistMusic {
+    pub contents: Contents,
+    pub header: Header,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Contents {
+    pub single_column_browse_results_renderer: ContentsRenderer<Tab>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Tab {
+    pub tab_renderer: ContentRenderer<SectionList>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SectionList {
+    /// Includes a continuation token for fetching recommendations
+    pub section_list_renderer: MusicContentsRenderer<ItemSection>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ItemSection {
+    #[serde(alias = "musicPlaylistShelfRenderer")]
+    pub music_shelf_renderer: MusicShelf,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct MusicShelf {
+    /// Playlist ID (only for playlists)
+    pub playlist_id: Option<String>,
+    #[serde_as(as = "VecSkipError<_>")]
+    pub contents: Vec<PlaylistMusicItem>,
+    /// Continuation token for fetching more (>100) playlist items
+    #[serde_as(as = "Option<VecSkipError<_>>")]
+    pub continuations: Option<Vec<MusicContinuation>>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct PlaylistMusicItem {
+    pub music_responsive_list_item_renderer: MusicItem,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Header {
+    pub music_detail_header_renderer: HeaderRenderer,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HeaderRenderer {
+    #[serde_as(as = "crate::serializer::text::Text")]
+    pub title: String,
+    /// Content type + Channel/Artist + Year.
+    /// Missing on artist_tracks view.
+    ///
+    /// `"Playlist", " • ", <"Best Music">, " • ", "2022"`
+    ///
+    /// `"Album", " • ", <"Helene Fischer">, " • ", "2021"`
+    pub subtitle: Option<Text>,
+    /// Playlist description. May contain hashtags which are
+    /// displayed as search links on the YouTube website.
+    #[serde_as(as = "Option<crate::serializer::text::Text>")]
+    pub description: Option<String>,
+    /// Playlist thumbnail / album cover.
+    /// Missing on artist_tracks view.
+    pub thumbnail: Option<MusicThumbnailRenderer>,
+    /// Number of tracks + playtime.
+    /// Missing on artist_tracks view.
+    ///
+    /// `"64 songs", " • ", "3 hours, 40 minutes"`
+    pub second_subtitle: Option<Text>,
+}

src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap

@@ -20,8 +20,9 @@ info:
     - url: "https://i.ytimg.com/vi_webp/pPvd8UxmSbQ/sddefault.webp"
       width: 640
       height: 480
-  channel_id: UCbxxEi-ImPlbLx5F-fHetEg
+  channel:
-  channel_name: RomanSenykMusic - Royalty Free Music
+    id: UCbxxEi-ImPlbLx5F-fHetEg
+    name: RomanSenykMusic - Royalty Free Music
   publish_date: ~
   view_count: 426567
   keywords:

src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap

@@ -23,8 +23,9 @@ info:
     - url: "https://i.ytimg.com/vi_webp/pPvd8UxmSbQ/maxresdefault.webp"
       width: 1920
       height: 1080
-  channel_id: UCbxxEi-ImPlbLx5F-fHetEg
+  channel:
-  channel_name: RomanSenykMusic - Royalty Free Music
+    id: UCbxxEi-ImPlbLx5F-fHetEg
+    name: RomanSenykMusic - Royalty Free Music
   publish_date: "2019-05-30T00:00:00Z"
   view_count: 426567
   keywords:

src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap

@@ -17,8 +17,9 @@ info:
     - url: "https://i.ytimg.com/vi/pPvd8UxmSbQ/hq720.jpg?sqp=-oaymwEXCNUGEOADIAQqCwjVARCqCBh4INgESFo&rs=AOn4CLA1wPf8NLXqHitgwoNBp4ydFCtDiA"
       width: 853
       height: 480
-  channel_id: UCbxxEi-ImPlbLx5F-fHetEg
+  channel:
-  channel_name: Romansenykmusic
+    id: UCbxxEi-ImPlbLx5F-fHetEg
+    name: Romansenykmusic
   publish_date: "2019-05-30T00:00:00Z"
   view_count: 426583
   keywords:

src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap

@@ -17,8 +17,9 @@ info:
     - url: "https://i.ytimg.com/vi/pPvd8UxmSbQ/sddefault.jpg"
       width: 640
       height: 480
-  channel_id: UCbxxEi-ImPlbLx5F-fHetEg
+  channel:
-  channel_name: RomanSenykMusic - Royalty Free Music
+    id: UCbxxEi-ImPlbLx5F-fHetEg
+    name: RomanSenykMusic - Royalty Free Music
   publish_date: ~
   view_count: 426567
   keywords:

src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap

@@ -23,8 +23,9 @@ info:
     - url: "https://i.ytimg.com/vi/pPvd8UxmSbQ/maxresdefault.jpg"
       width: 1920
       height: 1080
-  channel_id: UCbxxEi-ImPlbLx5F-fHetEg
+  channel:
-  channel_name: RomanSenykMusic - Royalty Free Music
+    id: UCbxxEi-ImPlbLx5F-fHetEg
+    name: RomanSenykMusic - Royalty Free Music
   publish_date: ~
   view_count: 426567
   keywords:

src/deobfuscate.rs

@@ -352,10 +352,16 @@ fn get_sts(player_js: &str) -> Result<String> {
 
 #[cfg(test)]
 mod tests {
+    use std::path::Path;
+
     use super::*;
     use test_log::test;
 
-    const TEST_JS: &str = include_str!("../notes/base.js");
+    const TEST_JS: Lazy<String> = Lazy::new(|| {
+        let js_path = Path::new("testfiles/deobf/dummy_player.js");
+        std::fs::read_to_string(js_path).unwrap()
+    });
+
     const N_DEOBF_FUNC: &str = r#"Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))},
 928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
 b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},
@@ -372,13 +378,13 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
 
     #[test]
     fn t_get_sig_fn_name() {
-        let dfunc_name = get_sig_fn_name(TEST_JS).unwrap();
+        let dfunc_name = get_sig_fn_name(&TEST_JS).unwrap();
         assert_eq!(dfunc_name, "Rva");
     }
 
     #[test]
     fn t_get_sig_fn() {
-        let dcode = get_sig_fn(TEST_JS).unwrap();
+        let dcode = get_sig_fn(&TEST_JS).unwrap();
         assert_eq!(
             dcode,
             r#"var qB={w8:function(a){a.reverse()},EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},Np:function(a,b){a.splice(0,b)}};var Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};var deobfuscate=Rva;"#
@@ -387,14 +393,14 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
 
     #[test]
     fn t_deobfuscate_sig() {
-        let dcode = get_sig_fn(TEST_JS).unwrap();
+        let dcode = get_sig_fn(&TEST_JS).unwrap();
         let deobf = deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i", &dcode).unwrap();
         assert_eq!(deobf, "AOq0QJ8wRAIgaryQHmplJ9xJSKFywyaSMHuuwZYsoMTfvRviG51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5f");
     }
 
     #[test]
     fn t_get_nsig_fn_name() {
-        let name = get_nsig_fn_name(TEST_JS).unwrap();
+        let name = get_nsig_fn_name(&TEST_JS).unwrap();
         assert_eq!(name, "Vo");
     }
 
@@ -435,13 +441,13 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
 
     #[test]
     fn t_get_nsig_fn() {
-        let res = get_nsig_fn(TEST_JS).unwrap();
+        let res = get_nsig_fn(&TEST_JS).unwrap();
         assert_eq!(res, N_DEOBF_FUNC);
     }
 
     #[test]
     fn t_get_sts() {
-        let res = get_sts(TEST_JS).unwrap();
+        let res = get_sts(&TEST_JS).unwrap();
         assert_eq!(res, "19187")
     }
 

src/download.rs

@@ -15,7 +15,7 @@ use tokio::{
 };
 
 use crate::{
-    model::{stream_filter::Filter, AudioCodec, FileFormat, PlayerData, VideoCodec},
+    model::{stream_filter::Filter, AudioCodec, FileFormat, VideoPlayer, VideoCodec},
     util,
 };
 
@@ -258,7 +258,7 @@ struct StreamDownload {
 }
 
 pub async fn download_video(
-    player_data: &PlayerData,
+    player_data: &VideoPlayer,
     output_dir: &str,
     output_fname: Option<String>,
     output_format: Option<String>,

src/model/mod.rs

@@ -1,5 +1,5 @@
-pub mod stream_filter;
 mod ordering;
+pub mod stream_filter;
 
 use std::ops::Range;
 
@@ -11,7 +11,7 @@ pub trait FileFormat {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct PlayerData {
+pub struct VideoPlayer {
     pub info: VideoInfo,
     pub video_streams: Vec<VideoStream>,
     pub video_only_streams: Vec<VideoStream>,
@@ -20,6 +20,17 @@ pub struct PlayerData {
     pub expires_in_seconds: u32,
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Playlist {
+    pub videos: Vec<Video>,
+    pub n_videos: u32,
+    pub ctoken: Option<String>,
+    pub name: String,
+    pub thumbnails: Vec<Thumbnail>,
+    pub description: Option<String>,
+    pub channel: Option<Channel>,
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct VideoInfo {
     pub id: String,
@@ -27,18 +38,13 @@ pub struct VideoInfo {
     pub description: Option<String>,
     pub length: u32,
     pub thumbnails: Vec<Thumbnail>,
-
+    pub channel: Channel,
-    pub channel_id: String,
-    pub channel_name: String,
-
     pub publish_date: Option<DateTime<Utc>>,
     pub view_count: u64,
     pub keywords: Vec<String>,
     pub category: Option<String>,
     pub is_live_content: bool,
     pub is_family_safe: Option<bool>,
-    // pub like_count: Option<u32>,
-    // pub dislike_count: Option<u32>
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
@@ -85,13 +91,13 @@ pub struct AudioStream {
 pub enum VideoCodec {
     #[default]
     Unknown,
-    /// MPEG-4 Part 14 https://en.wikipedia.org/wiki/MPEG-4_Part_14
+    /// MPEG-4 Part 14 <https://en.wikipedia.org/wiki/MPEG-4_Part_14>
     Mp4v,
-    /// avc1 aka H.264: https://en.wikipedia.org/wiki/Advanced_Video_Coding
+    /// avc1 aka H.264: <https://en.wikipedia.org/wiki/Advanced_Video_Coding>
     Avc1,
-    /// VP9: https://en.wikipedia.org/wiki/VP9
+    /// VP9: <https://en.wikipedia.org/wiki/VP9>
     Vp9,
-    /// AV1, the latest codec: https://en.wikipedia.org/wiki/AV1
+    /// AV1, the latest codec: <https://en.wikipedia.org/wiki/AV1>
     Av01,
 }
 
@@ -103,9 +109,9 @@ pub enum VideoCodec {
 pub enum AudioCodec {
     #[default]
     Unknown,
-    /// MP4A aka AAC: https://en.wikipedia.org/wiki/Advanced_Audio_Coding
+    /// MP4A aka AAC: <https://en.wikipedia.org/wiki/Advanced_Audio_Coding>
     Mp4a,
-    /// Opus: https://en.wikipedia.org/wiki/Opus_(audio_format)
+    /// Opus: <https://en.wikipedia.org/wiki/Opus_(audio_format)>
     Opus,
 }
 
@@ -175,3 +181,18 @@ pub struct Locale {
     pub lang: String,
     pub country: String,
 }
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct Video {
+    pub id: String,
+    pub title: String,
+    pub length: u32,
+    pub thumbnails: Vec<Thumbnail>,
+    pub channel: Channel,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct Channel {
+    pub id: String,
+    pub name: String,
+}

src/model/stream_filter.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 
 use super::{
-    AudioCodec, AudioFormat, AudioStream, PlayerData, VideoCodec, VideoFormat, VideoStream,
+    AudioCodec, AudioFormat, AudioStream, VideoCodec, VideoFormat, VideoPlayer, VideoStream,
 };
 
 #[derive(Debug, Default, Clone)]
@@ -108,7 +108,7 @@ impl Filter {
 
     /// Set the preferred audio language (2 letter ISO 639-1 code, e.g. `en`, `fr`).
     /// Some YouTube videos feature multiple audio streams in
-    /// different languages (e.g. https://www.youtube.com/watch?v=tVWWp1PqDus).
+    /// different languages (e.g. <https://www.youtube.com/watch?v=tVWWp1PqDus>).
     ///
     /// If this filter is unset or no stream matches,
     /// the filter returns the default audio stream.
@@ -232,7 +232,7 @@ impl Filter {
     }
 }
 
-impl PlayerData {
+impl VideoPlayer {
     pub fn select_audio_stream(&self, filter: &Filter) -> Option<&AudioStream> {
         let mut fallback: Option<&AudioStream> = None;
 
@@ -324,21 +324,25 @@ impl PlayerData {
 
 #[cfg(test)]
 mod tests {
+    use std::{fs::File, io::BufReader, path::Path};
+
     use super::*;
     use once_cell::sync::Lazy;
     use rstest::rstest;
     use velcro::hash_set;
 
-    const PLAYER_ML: Lazy<PlayerData> = Lazy::new(|| {
+    const PLAYER_ML: Lazy<VideoPlayer> = Lazy::new(|| {
-        serde_json::from_str::<PlayerData>(include_str!(
+        let json_path = Path::new("testfiles/player_model/multilanguage.json");
-            "../../testfiles/player_model/multilanguage.json"
+        let json_file = File::open(json_path).unwrap();
-        ))
+
-        .unwrap()
+        serde_json::from_reader(BufReader::new(json_file)).unwrap()
     });
 
-    const PLAYER_HDR: Lazy<PlayerData> = Lazy::new(|| {
+    const PLAYER_HDR: Lazy<VideoPlayer> = Lazy::new(|| {
-        serde_json::from_str::<PlayerData>(include_str!("../../testfiles/player_model/hdr.json"))
+        let json_path = Path::new("testfiles/player_model/hdr.json");
-            .unwrap()
+        let json_file = File::open(json_path).unwrap();
+
+        serde_json::from_reader(BufReader::new(json_file)).unwrap()
     });
 
     #[rstest]

src/serializer/text.rs

@@ -28,7 +28,7 @@ use serde_with::{serde_as, DefaultOnError, DeserializeAs};
 ///
 
 #[serde_as]
-#[derive(Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 #[serde(untagged)]
 pub enum Text {
     Simple {
@@ -70,6 +70,8 @@ pub enum TextLink {
     },
 }
 
+pub struct TextLinks {}
+
 #[derive(Deserialize)]
 struct TextLinkInternal {
     runs: Vec<TextLinkRun>,
@@ -150,6 +152,32 @@ pub enum PageType {
     Playlist,
 }
 
+fn map_text_linkrun(lr: &TextLinkRun) -> Option<TextLink> {
+    let text = lr.text.to_owned();
+    let nav = &lr.navigation_endpoint;
+
+    Some(match &nav.watch_endpoint {
+        Some(w) => TextLink::Video {
+            title: text,
+            video_id: w.video_id.to_owned(),
+        },
+        None => match &nav.browse_endpoint {
+            Some(b) => TextLink::Browse {
+                text,
+                page_type: match &b.browse_endpoint_context_supported_configs {
+                    Some(bc) => bc.browse_endpoint_context_music_config.page_type,
+                    None => match &nav.command_metadata {
+                        Some(cm) => cm.web_command_metadata.web_page_type,
+                        None => return None,
+                    },
+                },
+                browse_id: b.browse_id.to_owned(),
+            },
+            None => TextLink::None { text },
+        },
+    })
+}
+
 impl<'de> DeserializeAs<'de, TextLink> for TextLink {
     fn deserialize_as<D>(deserializer: D) -> Result<TextLink, D::Error>
     where
@@ -157,43 +185,35 @@ impl<'de> DeserializeAs<'de, TextLink> for TextLink {
     {
         let link = TextLinkInternal::deserialize(deserializer)?;
         if link.runs.len() != 1 {
-            return Err(serde::de::Error::invalid_length(link.runs.len(), &"1 run"));
+            return Err(serde::de::Error::invalid_length(
+                link.runs.len(),
+                &"1 run, use TextLinks for more",
+            ));
         }
 
-        let text = link.runs[0].text.to_owned();
+        Ok(some_or_bail!(
-        let nav = &link.runs[0].navigation_endpoint;
+            map_text_linkrun(&link.runs[0]),
+            Err(serde::de::Error::custom("missing/invalid browse endpoint"))
+        ))
+    }
+}
 
-        Ok(match &nav.watch_endpoint {
+impl<'de> DeserializeAs<'de, Vec<TextLink>> for TextLinks {
-            Some(w) => TextLink::Video {
+    fn deserialize_as<D>(deserializer: D) -> Result<Vec<TextLink>, D::Error>
-                title: text,
+    where
-                video_id: w.video_id.to_owned(),
+        D: Deserializer<'de>,
-            },
+    {
-            None => match &nav.browse_endpoint {
+        let link = TextLinkInternal::deserialize(deserializer)?;
-                Some(b) => TextLink::Browse {
+        Ok(link
-                    text,
+            .runs
-                    page_type: match &b.browse_endpoint_context_supported_configs {
+            .iter()
-                        Some(bc) => bc.browse_endpoint_context_music_config.page_type,
+            .filter_map(|r| map_text_linkrun(r))
-                        None => match &nav.command_metadata {
+            .collect())
-                            Some(cm) => cm.web_command_metadata.web_page_type,
-                            None => {
-                                return Err(serde::de::Error::custom(
-                                    "missing/invalid browse endpoint",
-                                ))
-                            }
-                        },
-                    },
-                    browse_id: b.browse_id.to_owned(),
-                },
-                None => TextLink::None { text },
-            },
-        })
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use crate::serializer::text::PageType;
-
     use super::TextLink;
     use rstest::rstest;
     use serde::Deserialize;
@@ -247,12 +267,19 @@ mod tests {
     }
 
     #[serde_as]
-    #[derive(Deserialize)]
+    #[derive(Debug, Deserialize)]
     struct SLink {
         #[serde_as(as = "crate::serializer::text::TextLink")]
         ln: TextLink,
     }
 
+    #[serde_as]
+    #[derive(Debug, Deserialize)]
+    struct SLinks {
+        #[serde_as(as = "crate::serializer::text::TextLinks")]
+        ln: Vec<TextLink>,
+    }
+
     #[test]
     fn t_link_video() {
         let test_json = r#"{
@@ -271,13 +298,14 @@ mod tests {
         }"#;
 
         let res = serde_json::from_str::<SLink>(&test_json).unwrap();
-
+        insta::assert_debug_snapshot!(res, @r###"
-        if let TextLink::Video { title, video_id } = res.ln {
+        SLink {
-            assert_eq!(title, "DEEP");
+            ln: Video {
-            assert_eq!(video_id, "wZIoIgz5mbs");
+                title: "DEEP",
-        } else {
+                video_id: "wZIoIgz5mbs",
-            panic!("not a video");
+            },
         }
+        "###);
     }
 
     #[test]
@@ -303,19 +331,15 @@ mod tests {
         }"#;
 
         let res = serde_json::from_str::<SLink>(&test_json).unwrap();
-
+        insta::assert_debug_snapshot!(res, @r###"
-        if let TextLink::Browse {
+        SLink {
-            text,
+            ln: Browse {
-            page_type,
+                text: "DEEP - The 1st Mini Album",
-            browse_id,
+                page_type: Album,
-        } = res.ln
+                browse_id: "MPREb_TKV2ccxsj5i",
-        {
+            },
-            assert_eq!(text, "DEEP - The 1st Mini Album");
-            assert_eq!(page_type, PageType::Album);
-            assert_eq!(browse_id, "MPREb_TKV2ccxsj5i");
-        } else {
-            panic!("not a browse item");
         }
+        "###);
     }
 
     #[test]
@@ -341,19 +365,15 @@ mod tests {
         }"#;
 
         let res = serde_json::from_str::<SLink>(&test_json).unwrap();
-
+        insta::assert_debug_snapshot!(res, @r###"
-        if let TextLink::Browse {
+        SLink {
-            text,
+            ln: Browse {
-            page_type,
+                text: "laserluca",
-            browse_id,
+                page_type: Channel,
-        } = res.ln
+                browse_id: "UCmxc6kXbU1J-0pR2F3wIx9A",
-        {
+            },
-            assert_eq!(text, "laserluca");
-            assert_eq!(page_type, PageType::Channel);
-            assert_eq!(browse_id, "UCmxc6kXbU1J-0pR2F3wIx9A");
-        } else {
-            panic!("not a browse item");
         }
+        "###);
     }
 
     #[test]
@@ -369,11 +389,72 @@ mod tests {
         }"#;
 
         let res = serde_json::from_str::<SLink>(&test_json).unwrap();
-
+        insta::assert_debug_snapshot!(res, @r###"
-        if let TextLink::None { text } = res.ln {
+        SLink {
-            assert_eq!(text, "Hello World");
+            ln: None {
-        } else {
+                text: "Hello World",
-            panic!("not none");
+            },
         }
+        "###);
+    }
+
+    #[test]
+    fn t_links_artists() {
+        let test_json = r#"{
+            "ln": {
+                "runs": [
+                    {
+                        "text": "Roland Kaiser",
+                        "navigationEndpoint": {
+                            "clickTrackingParams": "CNAMEMn0AhgFIhMI3aq914Tn-QIVi9ARCB3w6w_p",
+                            "browseEndpoint": {
+                                "browseId": "UCtqi0viP-suK-okUQfaw8Ew",
+                                    "browseEndpointContextSupportedConfigs": {
+                                    "browseEndpointContextMusicConfig": {
+                                        "pageType": "MUSIC_PAGE_TYPE_ARTIST"
+                                    }
+                                }
+                            }
+                        }
+                    },
+                    { "text": " & " },
+                    {
+                        "text": "Maite Kelly",
+                        "navigationEndpoint": {
+                            "clickTrackingParams": "CNAMEMn0AhgFIhMI3aq914Tn-QIVi9ARCB3w6w_p",
+                            "browseEndpoint": {
+                                "browseId": "UCY06CayCwdaOd1CnDgjy6uw",
+                                "browseEndpointContextSupportedConfigs": {
+                                    "browseEndpointContextMusicConfig": {
+                                        "pageType": "MUSIC_PAGE_TYPE_ARTIST"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                ]
+            }
+        }"#;
+
+        let res = serde_json::from_str::<SLinks>(&test_json).unwrap();
+        insta::assert_debug_snapshot!(res, @r###"
+        SLinks {
+            ln: [
+                Browse {
+                    text: "Roland Kaiser",
+                    page_type: Artist,
+                    browse_id: "UCtqi0viP-suK-okUQfaw8Ew",
+                },
+                None {
+                    text: " & ",
+                },
+                Browse {
+                    text: "Maite Kelly",
+                    page_type: Artist,
+                    browse_id: "UCY06CayCwdaOd1CnDgjy6uw",
+                },
+            ],
+        }
+        "###);
     }
 }

testfiles/deobf/dummy_player.js

@@ -0,0 +1,211 @@
+// This code is used to test the deobfuscation javascript extraction.
+// Since YouTube's player code is copyrighted, I can just copy-paste it
+// into my project.
+
+/*
+The filler javascript code comes from everything.js
+Source: https://github.com/michaelficarra/everything.js
+Copyright (c) 2014, Michael Ficarra, BSD 3-Clause License
+*/
+
+tab:for(;;)break	tab;
+verticalTab:for(;;)breakverticalTab;
+formFeed:for(;;)breakformFeed;
+space:for(;;)break space;
+nbsp:for(;;)break nbsp;
+bom:for(;;)breakbom;
+
+lineFeed:0
+0;
+carriageReturn:00;
+carriageReturnLineFeed:0
+0;
+lineSeparator:00;
+paragraphSeparator:00;
+
+var $, _, \u0078, x$, x_, x\u0030, xa, x0, x0a, x0123456789,
+  qwertyuiopasdfghjklzxcvbnm, QWERTYUIOPASDFGHJKLZXCVBNM;
+var œ一, ǻ둘, ɤ〩, φ, fiⅷ, ユニコード, x‌‍;
+
+null; true; false;
+
+0; 00; 1234567890; 01234567;
+0.; 0.00; 10.00; .0; .00
+0e0; 0E0; 0.e0; 0.00e+0; .00e-0;
+0x0; 0X0; 0x0123456789abcdefABCDEF;
+2e308;
+
+""; "'"; "\'\"\\\b\f\n\r\t\v\0";
+"\1\00\400\000";
+"\x01\x23\x45\x67\x89\xAB\xCD\xEF";
+"\u0123\u4567\u89AB\uCDEF"; "\
+";
+
+''; '"'; '\'\"\\\b\f\n\r\t\v\0';
+'\1\00\400\000';
+'\x01\x23\x45\x67\x89\xAB\xCD\xEF';
+'\u0123\u4567\u89AB\uCDEF'; '\
+';
+
+/x/; /|/; /|||/;
+/^$\b\B/; /(?=(?!(?:(.))))/;
+/a.\f\n\r\t\v\0\[\-\/\\\x00\u0000/; /\d\D\s\S\w\W/;
+/\ca\cb\cc\cd\ce\cf\cg\ch\ci\cj\ck\cl\cm\cn\co\cp\cq\cr\cs\ct\cu\cv\cw\cx\cy\cz/;
+/\cA\cB\cC\cD\cE\cF\cG\cH\cI\cJ\cK\cL\cM\cN\cO\cP\cQ\cR\cS\cT\cU\cV\cW\cX\cY\cZ/;
+/[a-z-]/; /[^\b\-^]/; /[/\]\\]/;
+/./i; /./g; /./m; /./igm;
+/.*/; /.*?/; /.+/; /.+?/; /.?/; /.??/;
+/.{0}/; /.{0,}/; /.{0,0}/;
+
+// STS
+{signatureTimestamp:19187};
+
+this;
+
+x;
+
+[]; [,]; [0]; [0,]; [,0]; [0,0]; [0,0,]; [0,,0]; [,,];
+
+({}); ({x:0}); ({x:0,y:0}); ({x:0,}); ({'x':0,"y":0,var:0,});
+({0:0}); ({0.:0}); ({0.0:0}); ({.0:0}); ({0e0:0}); ({0x0:0});
+({
+  get x(){}, set x(a){}, get 'y'(){}, set "y"(a){},
+  get 0(){}, set 0(a){}, get var(){}, set var(x){},
+});
+
+0..a;
+
+0[0];
+
+x = function f(){ return f; }; x[0] = x; x.a = x;
+
+new x(); new new x()();
+new x[0](); new x.a(); new x[0].a(); new x.a[0]();
+new x; new new x; new new x();
+new new x().a; new new x()[0];
+
+x(); x()(); x(x); x(x, x);
+x.a().a(); x[0]()[0](); x().a[0]();
+
+x++; x--;
+
+delete void typeof+-~!x; ++x; --x;
+
+0*0; 0/0; 0%0;
+
+0+0; 0-0;
+
+0<<0; 0>>0; 0>>>0;
+
+0<0; 0>0; 0<=0; 0>=0;
+0 instanceof function(){};
+0 in{};
+
+0==0; 0!=0; 0===0; 0!==0;
+
+0&0; 0^0; 0|0; 0&&0; 0||0;
+
+0?0:0; 0?0?0:0:0; 0||0?x=0:x=0;
+
+x=0; x*=0; x/=0; x%=0; x+=0; x-=0;
+x<<=0; x>>=0; x>>>=0; x&=0; x^=0; x|=0;
+
+0,0; 0,0,0; x=0,x=0;
+
+// Sig deobf heloper object
+
+var qB={w8:function(a){a.reverse()},
+EC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},
+Np:function(a,b){a.splice(0,b)}};
+
+{} {;} {0} {0;} {0;0} {0;0;}
+
+var x; var x,y; var x,y,z;
+var x=0; var x=0,y; var x,y=0; var x=0,y=0;
+
+;
+
+if(0); if(0);else;
+
+do;while(0);
+while(0);
+for(;;)break; for(0;0;0); for((0 in[]);0;);
+for(var a;;)break; for(var a,b;0;0);
+for(var a=0;;)break; for(var a=(0 in[]);0;);
+for(x in{}); for(var x in{});
+for(var x=[]in{}); for(var x=(0 in[])in{});
+
+debugger;
+
+// Sig deobf function
+var Rva;
+Rva=function(a){a=a.split("");qB.Np(a,3);qB.w8(a,41);qB.EC(a,55);qB.Np(a,3);qB.w8(a,33);qB.Np(a,3);qB.EC(a,48);qB.EC(a,17);qB.EC(a,43);return a.join("")};
+
+for(;0;)continue; x:for(;0;)continue x;
+
+for(;;)break; x:for(;;)break x;
+switch(0){case 0:break;}
+
+function f(){ return; }
+function f(){ return 0; }
+
+with(0);
+
+// n_deobf function
+{p.X&&(b=a.get("n"))&&(b=aF[0](c),a.set("n",b),vZ.length||Vo(""))};
+
+switch(0){} switch(0){case 0:} switch(0){case 0:case 0:}
+switch(0){default:} switch(0){case 0:default:case 0:}
+switch(0){case 0:;} switch(0){case 0:;;}
+switch(0){default:;} switch(0){default:;;}
+
+x:; x:y:;
+
+// n_deobf array_str
+var aF=[Vo];
+
+try { throw 0; }catch(x){}
+
+try{}catch(x){}
+try{}finally{}
+try{}catch(x){}finally{}
+
+// nsig deobf function
+Vo=function(a){var b=a.split(""),c=[function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(""))},
+928409064,-595856984,1403221911,653089124,-168714481,-1883008765,158931990,1346921902,361518508,1403221911,-362174697,-233641452,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
+b,158931990,791141857,-907319795,-1776185924,1595027902,-829736173,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},
+-1274951142,function(){for(var d=64,e=[];++d-e.length-32;){switch(d){case 91:d=44;continue;case 123:d=65;break;case 65:d-=18;continue;case 58:d=96;continue;case 46:d=95}e.push(String.fromCharCode(d))}return e},
+1758743891,function(d){d.reverse()},
+-830417133,"AF43j",1942017693,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)},
+null,-959991459,-287691724,-1365731946,b,1250397544,-1883008765,-1912322658,b,1300441121,null,-1962382380,1954679120,function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])},
+-985125467,function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())},
+null,497372841,-1912651541,function(d,e){d.push(e)},
+function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})},
+function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f}];
+c[30]=c;c[40]=c;c[46]=c;try{c[43](c[34]),c[45](c[40],c[47]),c[46](c[51],c[33]),c[16](c[47],c[36]),c[38](c[31],c[49]),c[16](c[11],c[39]),c[0](c[11]),c[35](c[0],c[30]),c[35](c[4],c[17]),c[34](c[48],c[7],c[11]()),c[35](c[4],c[23]),c[35](c[4],c[9]),c[5](c[48],c[28]),c[36](c[46],c[16]),c[4](c[41],c[1]),c[4](c[16],c[28]),c[3](c[40],c[17]),c[9](c[8],c[23]),c[45](c[30],c[4]),c[50](c[3],c[28]),c[36](c[51],c[23]),c[14](c[0],c[24]),c[14](c[35],c[1]),c[20](c[51],c[41]),c[15](c[8],c[0]),c[31](c[35]),c[29](c[26]),
+c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c[47],c[49]),c[1](c[44],c[28]),c[39](c[16]),c[32](c[42],c[22]),c[46](c[14],c[48]),c[26](c[29],c[10]),c[46](c[9],c[3]),c[32](c[45])}catch(d){return"enhanced_except_85UBjOr-_w8_"+a}return b.join("")};
+
+function f(){}
+function f(x){}
+function f(x,y){}
+function f(){ function f(){} }
+
+function f(){ "use strict" }
+function f(){ 'use strict' }
+function f(){ "other directive" }
+function f(){ 'other directive' }
+function f(){ ("string") }
+function f(){ ('string') }
+function f(){
+  'string'
+  +0
+}
+
+(function(){});
+(function(x){});
+(function(x,y){});
+(function(){ function f(){} });
+(function f(){});
+(function f(x){});
+(function f(x,y){});
+(function f(){ function f(){} });

testfiles/player_model/hdr.json

@@ -31,8 +31,7 @@
         "height": 1080
       }
     ],
-    "channel_id": "UCYq-iAOSZBvoUxvfzwKIZWA",
+    "channel": { "id": "UCYq-iAOSZBvoUxvfzwKIZWA", "name": "Jacob + Katie Schwarz" },
-    "channel_name": "Jacob + Katie Schwarz",
     "publish_date": "2018-06-12T00:00:00Z",
     "view_count": 216221243,
     "keywords": [
@@ -1126,4 +1125,4 @@
   ],
   "subtitles": [],
   "expires_in_seconds": 21540
-}
+}

testfiles/player_model/multilanguage.json

@@ -31,8 +31,7 @@
         "height": 1080
       }
     ],
-    "channel_id": "UCX6OQ3DkcsbYNE6H8uQQuVA",
+    "channel": { "id": "UCX6OQ3DkcsbYNE6H8uQQuVA", "name": "MrBeast" },
-    "channel_name": "MrBeast",
     "publish_date": "2022-07-23T00:00:00Z",
     "view_count": 71877575,
     "keywords": [],