diff --git a/notes/AB_Tests.md b/notes/AB_Tests.md index 94244c0..3af5c71 100644 --- a/notes/AB_Tests.md +++ b/notes/AB_Tests.md @@ -217,7 +217,7 @@ Short videos also have their own data models (`"reelItemRenderer"`). ![A/B test 3 screenshot](./_img/ab_3.png) Instead of subscriber count / video count, a channel item from the search result now -displays the channel handle and the subscriber count. +displays the channel handle and the subscriber count. The video count was removed. The implementation looks pretty quick and dirty, as they did not even bother to rename their response parameters. So this might change again in the future. diff --git a/src/client/channel.rs b/src/client/channel.rs index 036b6ea..f140cec 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -142,7 +142,7 @@ impl MapResponse>> for response::Channel { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data: self.response_context.visitor_data, + visitor_data: self.response_context.visitor_data.clone(), has_shorts: content.has_shorts, has_live: content.has_live, }, @@ -157,7 +157,13 @@ impl MapResponse>> for response::Channel { mapper.map_response(items); MapResult { - c: Paginator::new(None, mapper.items, mapper.ctoken), + c: Paginator::new_ext( + None, + mapper.items, + mapper.ctoken, + self.response_context.visitor_data, + crate::param::ContinuationEndpoint::Browse, + ), warnings: mapper.warnings, } } diff --git a/src/client/music_details.rs b/src/client/music_details.rs index e30a94f..07ae014 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -166,7 +166,9 @@ impl MapResponse for response::MusicDetails { } } - let content = content.ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?; + let content = content.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed( + "track not found", + )))?; let track_item = content .contents .c @@ -215,7 +217,9 @@ impl MapResponse> for response::MusicDetails { let content = tabs .into_iter() .find_map(|t| t.tab_renderer.content) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))? + .ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed( + "radio unavailable", + )))? .music_queue_renderer .content .playlist_panel_renderer; @@ -261,10 +265,15 @@ impl MapResponse for response::MusicLyrics { let lyrics = self .contents .section_list_renderer - .contents - .into_iter() - .find_map(|item| item.music_description_shelf_renderer) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("no content")))?; + .and_then(|sl| { + sl.contents + .into_iter() + .find_map(|item| item.music_description_shelf_renderer) + }) + .ok_or(match self.contents.message_renderer { + Some(msg) => ExtractionError::ContentUnavailable(Cow::Owned(msg.text)), + None => ExtractionError::InvalidData(Cow::Borrowed("no content")), + })?; Ok(MapResult { c: Lyrics { diff --git a/src/client/player.rs b/src/client/player.rs index c8d0c71..27ec7b8 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -154,7 +154,7 @@ impl MapResponse for response::Player { // reason: "This video requires payment to watch." "payment" => return Err(ExtractionError::VideoUnavailable("DRM", reason)), // reason: "The uploader has not made this video available in your country." - "country" => return Err(ExtractionError::VideoGeoblock), + "country" => return Err(ExtractionError::VideoGeoblocked), // reason (Android): "This video can only be played on newer versions of Android or other supported devices." // reason (TV client): "Playback on other websites has been disabled by the video owner." "Android" | "websites" => { diff --git a/src/client/response/music_details.rs b/src/client/response/music_details.rs index 5407345..0d7e6d2 100644 --- a/src/client/response/music_details.rs +++ b/src/client/response/music_details.rs @@ -1,8 +1,11 @@ use serde::Deserialize; use serde_with::serde_as; +use serde_with::DefaultOnError; use crate::serializer::text::Text; +use super::AlertRenderer; +use super::ContentsRenderer; use super::{ music_item::{ItemSection, PlaylistPanelRenderer}, ContentRenderer, SectionList, @@ -46,9 +49,12 @@ pub(crate) struct Tab { } /// Watch next tab +#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TabRenderer { + #[serde(default)] + #[serde_as(as = "DefaultOnError")] pub content: Option, pub endpoint: Option, } @@ -101,12 +107,19 @@ pub(crate) struct PlaylistPanel { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct MusicLyrics { - pub contents: SectionList, + pub contents: LyricsContents, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct LyricsContents { + pub message_renderer: Option, + pub section_list_renderer: Option>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct LyricsSection { pub music_description_shelf_renderer: Option, } diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_shorts.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_shorts.snap index bae5d77..0200294 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_shorts.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_shorts.snap @@ -1369,6 +1369,7 @@ Channel( ), ], ctoken: Some("4qmFsgLlARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGsgBOGdhU0FScVBBVktNQVFxSEFRcGZRME00VVVGU2IyWnZaMWxqUTJob1ZsRXlaelJhTUdoclpFaHdVRTF1VWxsYVJGVTFUVEU1YVdGclZubFdNbU5SUVZOSlVrTm5PSGhQYWtVeVRtcFpOVTlFWnpCTmFsRjVUbFJyY1VSUmIweFRXRXBUVFROU1ZWZ3diSGhYYldNU0pEWXpOakl6WkdRd0xUQXdNREF0TW1GaFlpMWlZalF6TFRVNE1qUXlPV00yTmpFell4Z0I%3D"), + visitor_data: Some("Cgt3cDF3NkYwWWlENCiI8_CaBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap index 0bb35f9..b0a4459 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap @@ -1351,6 +1351,7 @@ Channel( ), ], ctoken: Some("4qmFsgKhARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGoQBOGdaZ0dsNTZYQXBZQ2pCRlozTkpiRXRwYURZdFNGZG9ZbVZuUVZObmVVMUJSVFJJYTBsTVExQlFVbXR3YjBkRlMwUnBkVmRhU1VGV1FVRVNKRFl6TkRnd056Wm1MVEF3TURBdE1tRXhaUzA0TnpRNUxXUTBaalUwTjJZNE16a3pZeGdC"), + visitor_data: Some("CgtQdE9zVVR3NVBDbyjz0ZKaBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap index 551d8e1..6cd0789 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap @@ -1380,6 +1380,7 @@ Channel( ), ], ctoken: Some("4qmFsgKhARIYVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RGoQBOGdaZ0dsNTZYQXBZQ2pCRlozTkpOV054YjJ4eWNYSnBjeTA0UVZObmVVMUJSVFJJYTBsTVEwbEhjV3cxYjBkRlVHcHlYM2hTU1VGV1FVRVNKRFl6TkRZMVlqZzFMVEF3TURBdE1qTXlOaTA1WTJSbUxUTmpNamcyWkRReU1tWTNOaGdC"), + visitor_data: Some("Cgs4ZFVmMzVlU1dxbyiBqpeaBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap index e13381c..963406c 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap @@ -1380,6 +1380,7 @@ Channel( ), ], ctoken: Some("4qmFsgKrARIYVUMyRGpGRTdYZjExVVJacVdCaWdjVk9RGmBFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNRVZuYjBsMVMzWlpPVXREWWw5TVRraExSRWwzUVZSblpWRm5kMGx3ZFVOMGJWRlpVVjlOUjA1d2QwcEpRVlpCUVElM0QlM0SaAixicm93c2UtZmVlZFVDMkRqRkU3WGYxMVVSWnFXQmlnY1ZPUXZpZGVvczEwMg%3D%3D"), + visitor_data: Some("CgszNU5rbDVZS2hMcyim4K2ZBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap index 945fe0b..20402d0 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap @@ -995,6 +995,7 @@ Channel( ), ], ctoken: None, + visitor_data: Some("CgtkYXJITElwYmd4OCj85a2ZBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap index c2a8a42..5430e3e 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap @@ -991,6 +991,7 @@ Channel( ), ], ctoken: Some("4qmFsgKrARIYVUNoOGdIZHR6TzJ0WGQ1OTNfYmpFcldnGmBFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNRVZuYzBrM2NsTnVkazF4U1hWemRqQkJVMmQ1VFVGRk5FaHJTVXhEVUhac2NscHJSMFZKWVhWd016RkpRVlpCUVElM0QlM0SaAixicm93c2UtZmVlZFVDaDhnSGR0ek8ydFhkNTkzX2JqRXJXZ3ZpZGVvczEwMg%3D%3D"), + visitor_data: Some("CgtneXVRbGtSMWtlYyj75a2ZBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap index b4a20a4..df5f371 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap @@ -1368,6 +1368,7 @@ Channel( ), ], ctoken: Some("4qmFsgKnARIYVUNjdmZIYS1HSFNPSEZBalUwLUllNTdBGlxFZ1oyYVdSbGIzTVlBeUFBTUFFNEFlb0RNa1ZuYzBsdFlYbFhlRkJZYnpWMlltcEJVMmQ1VFVGRk5FaHJTVTFEVFdGVWVrcHJSMFZPUzJzMmNqQkNVMEZHVVVGQpoCLGJyb3dzZS1mZWVkVUNjdmZIYS1HSFNPSEZBalUwLUllNTdBdmlkZW9zMTAy"), + visitor_data: Some("Cgs4Ri1tLW1KNWozNCjGk8yZBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/error.rs b/src/error.rs index d514e26..ced5358 100644 --- a/src/error.rs +++ b/src/error.rs @@ -74,7 +74,7 @@ pub enum ExtractionError { #[error("Video is age restricted")] VideoAgeRestricted, #[error("Video is not available in your country")] - VideoGeoblock, + VideoGeoblocked, #[error("Video cant be played with this client. Reason (from YT): {0}")] VideoClientUnsupported(String), #[error("Content is not available. Reason: {0}")] diff --git a/tests/snapshots/youtube__music_lyrics.snap b/tests/snapshots/youtube__music_lyrics.snap index ecb76ac..5d233fa 100644 --- a/tests/snapshots/youtube__music_lyrics.snap +++ b/tests/snapshots/youtube__music_lyrics.snap @@ -3,6 +3,6 @@ source: tests/youtube.rs expression: lyrics --- Lyrics( - body: "Eyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nWon\'t you let the light shine through\n\nEyes, in the sky, gazing far into the night\nI raise my hand to the fire, but it\'s no use\n\'Cause you can\'t stop it from shining through\nIt\'s true\nBaby let the light shine through\nIf you believe it\'s true\nBaby won\'t you let the light shine through\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you\nFor you", + body: "Still and silent\nCalm before the storm\nGold and diamond\nJewels behind the throne\n\nInto the night\nOut of the dark\nTake to the sky\nChasing the stars\nAll that we said\nAll that we are\nWaiting to fly\nThis is the start\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nStone and feather\nMove outside your head\nNow or never\nStrong in every step\n\nGive me a sign\nHitting the mark\nTake to the sky\nChasing the stars\nOpen your eyes\nWatching afar\nWaiting to fly\nThis is the start\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria\n\nHide and seek\nReason and rhyme\nGrand and glorious\nLiving the dream\nYours and mine\nEuphoria", footer: "Source: Musixmatch", ) diff --git a/tests/youtube.rs b/tests/youtube.rs index 7e12017..ea980b3 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -460,6 +460,7 @@ async fn get_video_details() { assert!(!details.is_live); assert!(!details.is_ccommons); + assert!(details.recommended.visitor_data.is_some()); assert_next(details.recommended, &rp.query(), 10, 2).await; assert_gte(details.top_comments.count.unwrap(), 700_000, "comments"); @@ -497,6 +498,7 @@ async fn get_video_details_music() { assert!(!details.is_live); assert!(!details.is_ccommons); + assert!(details.recommended.visitor_data.is_some()); assert_next(details.recommended, &rp.query(), 10, 2).await; // Update(01.11.2022): comments are sometimes enabled @@ -545,6 +547,7 @@ async fn get_video_details_ccommons() { assert!(!details.is_live); assert!(details.is_ccommons); + assert!(details.recommended.visitor_data.is_some()); assert_next(details.recommended, &rp.query(), 10, 2).await; assert_gte(details.top_comments.count.unwrap(), 2199, "comments"); @@ -669,6 +672,7 @@ async fn get_video_details_chapters() { "###); } + assert!(details.recommended.visitor_data.is_some()); assert_next(details.recommended, &rp.query(), 10, 2).await; assert_gte(details.top_comments.count.unwrap(), 3200, "comments"); @@ -713,6 +717,7 @@ async fn get_video_details_live() { assert!(details.is_live); assert!(!details.is_ccommons); + assert!(details.recommended.visitor_data.is_some()); assert_next(details.recommended, &rp.query(), 10, 2).await; // No comments because livestream @@ -787,6 +792,7 @@ async fn get_video_comments() { .unwrap(); assert_gte(top_comments.items.len(), 10, "comments"); assert!(!top_comments.is_exhausted()); + assert!(top_comments.visitor_data.is_some()); let n_comments = top_comments.count.unwrap(); assert_gte(n_comments, 700_000, "comments"); @@ -805,6 +811,7 @@ async fn get_video_comments() { .unwrap(); assert_gte(latest_comments.items.len(), 10, "next comments"); assert!(!latest_comments.is_exhausted()); + assert!(latest_comments.visitor_data.is_some()); } //#CHANNEL @@ -1233,6 +1240,9 @@ async fn startpage() { let rp = RustyPipe::builder().strict().build(); let startpage = rp.query().startpage().await.unwrap(); + // The startpage requires visitor data to fetch continuations + assert!(startpage.visitor_data.is_some()); + assert_next(startpage, &rp.query(), 20, 2).await; } @@ -1800,7 +1810,7 @@ async fn music_details(#[case] name: &str, #[case] id: &str) { #[tokio::test] async fn music_lyrics() { let rp = RustyPipe::builder().strict().build(); - let track = rp.query().music_details("n4tK7LYFxI0").await.unwrap(); + let track = rp.query().music_details("NO8Arj4yeww").await.unwrap(); let lyrics = rp .query() .music_lyrics(&track.lyrics_id.unwrap()) @@ -1809,6 +1819,26 @@ async fn music_lyrics() { insta::assert_ron_snapshot!(lyrics); } +#[tokio::test] +async fn music_lyrics_not_found() { + let rp = RustyPipe::builder().strict().build(); + let track = rp.query().music_details("ekXI8qrbe1s").await.unwrap(); + let err = rp + .query() + .music_lyrics(&track.lyrics_id.unwrap()) + .await + .unwrap_err(); + + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); +} + #[rstest] #[case::a("7nigXQS1Xb0", true)] #[case::b("4t3SUDZCBaQ", false)] @@ -1914,6 +1944,21 @@ async fn music_related(#[case] id: &str, #[case] full: bool) { } } +#[tokio::test] +async fn music_details_not_found() { + let rp = RustyPipe::builder().strict().build(); + let err = rp.query().music_details("7nigXQS1XbZ").await.unwrap_err(); + + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); +} + #[tokio::test] async fn music_radio_track() { let rp = RustyPipe::builder().strict().build(); @@ -1921,6 +1966,25 @@ async fn music_radio_track() { assert_next(tracks, &rp.query(), 20, 3).await; } +#[tokio::test] +async fn music_radio_track_not_found() { + let rp = RustyPipe::builder().strict().build(); + let err = rp + .query() + .music_radio_track("7nigXQS1XbZ") + .await + .unwrap_err(); + + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); +} + #[tokio::test] async fn music_radio_playlist() { let rp = RustyPipe::builder().strict().build(); @@ -1932,6 +1996,27 @@ async fn music_radio_playlist() { assert_next(tracks, &rp.query(), 10, 1).await; } +#[tokio::test] +async fn music_radio_playlist_not_found() { + let rp = RustyPipe::builder().strict().build(); + let res = rp + .query() + .music_radio_playlist("PL5dDx681T4bR7ZF1IuWzOv1omlZZZZZZZ") + .await; + + // Currently this returns valid data + if let Err(err) = res { + assert!( + matches!( + err, + Error::Extraction(ExtractionError::ContentUnavailable(_)) + ), + "got: {}", + err + ); + } +} + //#AB TESTS const VISITOR_DATA_SEARCH_CHANNEL_HANDLES: &str = "CgszYlc1Yk1WZGRCSSjrwOSbBg%3D%3D";