fix: lyrics not found error, no channel shorts cont

notes/AB_Tests.md

@@ -217,7 +217,7 @@ Short videos also have their own data models (`"reelItemRenderer"`).
 ![A/B test 3 screenshot](./_img/ab_3.png)
 
 Instead of subscriber count / video count, a channel item from the search result now
-displays the channel handle and the subscriber count.
+displays the channel handle and the subscriber count. The video count was removed.
 
 The implementation looks pretty quick and dirty, as they did not even bother to rename
 their response parameters. So this might change again in the future.

src/client/channel.rs

@@ -142,7 +142,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
                 header: self.header,
                 metadata: self.metadata,
                 microformat: self.microformat,
-                visitor_data: self.response_context.visitor_data,
+                visitor_data: self.response_context.visitor_data.clone(),
                 has_shorts: content.has_shorts,
                 has_live: content.has_live,
             },
@@ -157,7 +157,13 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
                 mapper.map_response(items);
 
                 MapResult {
-                    c: Paginator::new(None, mapper.items, mapper.ctoken),
+                    c: Paginator::new_ext(
+                        None,
+                        mapper.items,
+                        mapper.ctoken,
+                        self.response_context.visitor_data,
+                        crate::param::ContinuationEndpoint::Browse,
+                    ),
                     warnings: mapper.warnings,
                 }
             }

src/client/music_details.rs

@@ -166,7 +166,9 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
             }
         }
 
-        let content = content.ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?;
+        let content = content.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
+            "track not found",
+        )))?;
         let track_item = content
             .contents
             .c
@@ -215,7 +217,9 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
         let content = tabs
             .into_iter()
             .find_map(|t| t.tab_renderer.content)
-            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?
+            .ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
+                "radio unavailable",
+            )))?
             .music_queue_renderer
             .content
             .playlist_panel_renderer;
@@ -261,10 +265,15 @@ impl MapResponse<Lyrics> for response::MusicLyrics {
         let lyrics = self
             .contents
             .section_list_renderer
-            .contents
-            .into_iter()
-            .find_map(|item| item.music_description_shelf_renderer)
-            .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?;
+            .and_then(|sl| {
+                sl.contents
+                    .into_iter()
+                    .find_map(|item| item.music_description_shelf_renderer)
+            })
+            .ok_or(match self.contents.message_renderer {
+                Some(msg) => ExtractionError::ContentUnavailable(Cow::Owned(msg.text)),
+                None => ExtractionError::InvalidData(Cow::Borrowed("no content")),
+            })?;
 
         Ok(MapResult {
             c: Lyrics {

src/client/player.rs

@@ -154,7 +154,7 @@ impl MapResponse<VideoPlayer> for response::Player {
                         // reason: "This video requires payment to watch."
                         "payment" => return Err(ExtractionError::VideoUnavailable("DRM", reason)),
                         // reason: "The uploader has not made this video available in your country."
-                        "country" => return Err(ExtractionError::VideoGeoblock),
+                        "country" => return Err(ExtractionError::VideoGeoblocked),
                         // reason (Android): "This video can only be played on newer versions of Android or other supported devices."
                         // reason (TV client): "Playback on other websites has been disabled by the video owner."
                         "Android" | "websites" => {

src/client/response/music_details.rs

@@ -1,8 +1,11 @@
 use serde::Deserialize;
 use serde_with::serde_as;
+use serde_with::DefaultOnError;
 
 use crate::serializer::text::Text;
 
+use super::AlertRenderer;
+use super::ContentsRenderer;
 use super::{
     music_item::{ItemSection, PlaylistPanelRenderer},
     ContentRenderer, SectionList,
@@ -46,9 +49,12 @@ pub(crate) struct Tab {
 }
 
 /// Watch next tab
+#[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct TabRenderer {
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
     pub content: Option<TabContent>,
     pub endpoint: Option<TabEndpoint>,
 }
@@ -101,12 +107,19 @@ pub(crate) struct PlaylistPanel {
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct MusicLyrics {
-    pub contents: SectionList<LyricsContents>,
+    pub contents: LyricsContents,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct LyricsContents {
+    pub message_renderer: Option<AlertRenderer>,
+    pub section_list_renderer: Option<ContentsRenderer<LyricsSection>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct LyricsSection {
     pub music_description_shelf_renderer: Option<LyricsRenderer>,
 }
 

src/client/snapshots/rustypipe__client__channel__tests__map_channel_shorts.snap

@@ -1369,6 +1369,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgLlARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGsgBOGdhU0FScVBBVktNQVFxSEFRcGZRME00VVVGU2IyWnZaMWxqUTJob1ZsRXlaelJhTUdoclpFaHdVRTF1VWxsYVJGVTFUVEU1YVdGclZubFdNbU5SUVZOSlVrTm5PSGhQYWtVeVRtcFpOVTlFWnpCTmFsRjVUbFJyY1VSUmIweFRXRXBUVFROU1ZWZ3diSGhYYldNU0pEWXpOakl6WkdRd0xUQXdNREF0TW1GaFlpMWlZalF6TFRVNE1qUXlPV00yTmpFell4Z0I%3D"),
+    visitor_data: Some("Cgt3cDF3NkYwWWlENCiI8_CaBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap

@@ -1351,6 +1351,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgKhARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGoQBOGdaZ0dsNTZYQXBZQ2pCRlozTkpiRXRwYURZdFNGZG9ZbVZuUVZObmVVMUJSVFJJYTBsTVExQlFVbXR3YjBkRlMwUnBkVmRhU1VGV1FVRVNKRFl6TkRnd056Wm1MVEF3TURBdE1tRXhaUzA0TnpRNUxXUTBaalUwTjJZNE16a3pZeGdC"),
+    visitor_data: Some("CgtQdE9zVVR3NVBDbyjz0ZKaBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap

@@ -1380,6 +1380,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgKhARIYVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RGoQBOGdaZ0dsNTZYQXBZQ2pCRlozTkpOV054YjJ4eWNYSnBjeTA0UVZObmVVMUJSVFJJYTBsTVEwbEhjV3cxYjBkRlVHcHlYM2hTU1VGV1FVRVNKRFl6TkRZMVlqZzFMVEF3TURBdE1qTXlOaTA1WTJSbUxUTmpNamcyWkRReU1tWTNOaGdC"),
+    visitor_data: Some("Cgs4ZFVmMzVlU1dxbyiBqpeaBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap

@@ -1380,6 +1380,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgKrARIYVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RGmBFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNRVZuYjBsMVMzWlpPVXREWWw5TVRraExSRWwzUVZSblpWRm5kMGx3ZFVOMGJWRlpVVjlOUjA1d2QwcEpRVlpCUVElM0QlM0SaAixicm93c2UtZmVlZFVDMkRqRkU3WGYxMVVSWnFXQmlnY1ZPUXZpZGVvczEwMg%3D%3D"),
+    visitor_data: Some("CgszNU5rbDVZS2hMcyim4K2ZBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap

@@ -995,6 +995,7 @@ Channel(
       ),
     ],
     ctoken: None,
+    visitor_data: Some("CgtkYXJITElwYmd4OCj85a2ZBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap

@@ -991,6 +991,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgKrARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGmBFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNRVZuYzBrM2NsTnVkazF4U1hWemRqQkJVMmQ1VFVGRk5FaHJTVXhEVUhac2NscHJSMFZKWVhWd016RkpRVlpCUVElM0QlM0SaAixicm93c2UtZmVlZFVDaDhnSGR0ek8ydFhkNTkzX2JqRXJXZ3ZpZGVvczEwMg%3D%3D"),
+    visitor_data: Some("CgtneXVRbGtSMWtlYyj75a2ZBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap

@@ -1368,6 +1368,7 @@ Channel(
       ),
     ],
     ctoken: Some("4qmFsgKnARIYVUNjdmZIYS1HSFNPSEZBalUwLUllNTdBGlxFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNa1ZuYzBsdFlYbFhlRkJZYnpWMlltcEJVMmQ1VFVGRk5FaHJTVTFEVFdGVWVrcHJSMFZPUzJzMmNqQkNVMEZHVVVGQpoCLGJyb3dzZS1mZWVkVUNjdmZIYS1HSFNPSEZBalUwLUllNTdBdmlkZW9zMTAy"),
+    visitor_data: Some("Cgs4Ri1tLW1KNWozNCjGk8yZBg%3D%3D"),
     endpoint: browse,
   ),
 )

src/error.rs

@@ -74,7 +74,7 @@ pub enum ExtractionError {
     #[error("Video is age restricted")]
     VideoAgeRestricted,
     #[error("Video is not available in your country")]
-    VideoGeoblock,
+    VideoGeoblocked,
     #[error("Video cant be played with this client. Reason (from YT): {0}")]
     VideoClientUnsupported(String),
     #[error("Content is not available. Reason: {0}")]

tests/snapshots/youtube__music_lyrics.snap

@@ -3,6 +3,6 @@ source: tests/youtube.rs
 expression: lyrics
 ---
 Lyrics(
-  body: "Eyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nWon\'t you let the light shine through\n\nEyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you",
+  body: "Still and silent\nCalm before the storm\nGold and diamond\nJewels behind the throne\n\nInto the night\nOut of the dark\nTake to the sky\nChasing the stars\nAll that we said\nAll that we are\nWaiting to fly\nThis is the start\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nStone and feather\nMove outside your head\nNow or never\nStrong in every step\n\nGive me a sign\nHitting the mark\nTake to the sky\nChasing the stars\nOpen your eyes\nWatching afar\nWaiting to fly\nThis is the start\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria",
   footer: "Source: Musixmatch",
 )

tests/youtube.rs

@@ -460,6 +460,7 @@ async fn get_video_details() {
     assert!(!details.is_live);
     assert!(!details.is_ccommons);
 
+    assert!(details.recommended.visitor_data.is_some());
     assert_next(details.recommended, &rp.query(), 10, 2).await;
 
     assert_gte(details.top_comments.count.unwrap(), 700_000, "comments");
@@ -497,6 +498,7 @@ async fn get_video_details_music() {
     assert!(!details.is_live);
     assert!(!details.is_ccommons);
 
+    assert!(details.recommended.visitor_data.is_some());
     assert_next(details.recommended, &rp.query(), 10, 2).await;
 
     // Update(01.11.2022): comments are sometimes enabled
@@ -545,6 +547,7 @@ async fn get_video_details_ccommons() {
     assert!(!details.is_live);
     assert!(details.is_ccommons);
 
+    assert!(details.recommended.visitor_data.is_some());
     assert_next(details.recommended, &rp.query(), 10, 2).await;
 
     assert_gte(details.top_comments.count.unwrap(), 2199, "comments");
@@ -669,6 +672,7 @@ async fn get_video_details_chapters() {
         "###);
     }
 
+    assert!(details.recommended.visitor_data.is_some());
     assert_next(details.recommended, &rp.query(), 10, 2).await;
 
     assert_gte(details.top_comments.count.unwrap(), 3200, "comments");
@@ -713,6 +717,7 @@ async fn get_video_details_live() {
     assert!(details.is_live);
     assert!(!details.is_ccommons);
 
+    assert!(details.recommended.visitor_data.is_some());
     assert_next(details.recommended, &rp.query(), 10, 2).await;
 
     // No comments because livestream
@@ -787,6 +792,7 @@ async fn get_video_comments() {
         .unwrap();
     assert_gte(top_comments.items.len(), 10, "comments");
     assert!(!top_comments.is_exhausted());
+    assert!(top_comments.visitor_data.is_some());
 
     let n_comments = top_comments.count.unwrap();
     assert_gte(n_comments, 700_000, "comments");
@@ -805,6 +811,7 @@ async fn get_video_comments() {
         .unwrap();
     assert_gte(latest_comments.items.len(), 10, "next comments");
     assert!(!latest_comments.is_exhausted());
+    assert!(latest_comments.visitor_data.is_some());
 }
 
 //#CHANNEL
@@ -1233,6 +1240,9 @@ async fn startpage() {
     let rp = RustyPipe::builder().strict().build();
     let startpage = rp.query().startpage().await.unwrap();
 
+    // The startpage requires visitor data to fetch continuations
+    assert!(startpage.visitor_data.is_some());
+
     assert_next(startpage, &rp.query(), 20, 2).await;
 }
 
@@ -1800,7 +1810,7 @@ async fn music_details(#[case] name: &str, #[case] id: &str) {
 #[tokio::test]
 async fn music_lyrics() {
     let rp = RustyPipe::builder().strict().build();
-    let track = rp.query().music_details("n4tK7LYFxI0").await.unwrap();
+    let track = rp.query().music_details("NO8Arj4yeww").await.unwrap();
     let lyrics = rp
         .query()
         .music_lyrics(&track.lyrics_id.unwrap())
@@ -1809,6 +1819,26 @@ async fn music_lyrics() {
     insta::assert_ron_snapshot!(lyrics);
 }
 
+#[tokio::test]
+async fn music_lyrics_not_found() {
+    let rp = RustyPipe::builder().strict().build();
+    let track = rp.query().music_details("ekXI8qrbe1s").await.unwrap();
+    let err = rp
+        .query()
+        .music_lyrics(&track.lyrics_id.unwrap())
+        .await
+        .unwrap_err();
+
+    assert!(
+        matches!(
+            err,
+            Error::Extraction(ExtractionError::ContentUnavailable(_))
+        ),
+        "got: {}",
+        err
+    );
+}
+
 #[rstest]
 #[case::a("7nigXQS1Xb0", true)]
 #[case::b("4t3SUDZCBaQ", false)]
@@ -1914,6 +1944,21 @@ async fn music_related(#[case] id: &str, #[case] full: bool) {
     }
 }
 
+#[tokio::test]
+async fn music_details_not_found() {
+    let rp = RustyPipe::builder().strict().build();
+    let err = rp.query().music_details("7nigXQS1XbZ").await.unwrap_err();
+
+    assert!(
+        matches!(
+            err,
+            Error::Extraction(ExtractionError::ContentUnavailable(_))
+        ),
+        "got: {}",
+        err
+    );
+}
+
 #[tokio::test]
 async fn music_radio_track() {
     let rp = RustyPipe::builder().strict().build();
@@ -1921,6 +1966,25 @@ async fn music_radio_track() {
     assert_next(tracks, &rp.query(), 20, 3).await;
 }
 
+#[tokio::test]
+async fn music_radio_track_not_found() {
+    let rp = RustyPipe::builder().strict().build();
+    let err = rp
+        .query()
+        .music_radio_track("7nigXQS1XbZ")
+        .await
+        .unwrap_err();
+
+    assert!(
+        matches!(
+            err,
+            Error::Extraction(ExtractionError::ContentUnavailable(_))
+        ),
+        "got: {}",
+        err
+    );
+}
+
 #[tokio::test]
 async fn music_radio_playlist() {
     let rp = RustyPipe::builder().strict().build();
@@ -1932,6 +1996,27 @@ async fn music_radio_playlist() {
     assert_next(tracks, &rp.query(), 10, 1).await;
 }
 
+#[tokio::test]
+async fn music_radio_playlist_not_found() {
+    let rp = RustyPipe::builder().strict().build();
+    let res = rp
+        .query()
+        .music_radio_playlist("PL5dDx681T4bR7ZF1IuWzOv1omlZZZZZZZ")
+        .await;
+
+    // Currently this returns valid data
+    if let Err(err) = res {
+        assert!(
+            matches!(
+                err,
+                Error::Extraction(ExtractionError::ContentUnavailable(_))
+            ),
+            "got: {}",
+            err
+        );
+    }
+}
+
 //#AB TESTS
 
 const VISITOR_DATA_SEARCH_CHANNEL_HANDLES: &str = "CgszYlc1Yk1WZGRCSSjrwOSbBg%3D%3D";