fix: send visitor data for YTM playlists

src/client/channel.rs

@@ -238,7 +238,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
             mapper.items,
             mapper.ctoken,
             visitor_data,
-            crate::model::paginator::ContinuationEndpoint::Browse,
+            ContinuationEndpoint::Browse,
         );
 
         Ok(MapResult {

src/client/mod.rs

@@ -1185,8 +1185,14 @@ impl RustyPipeQuery {
     /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
     /// - `method`: HTTP method
     /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
-    async fn request_builder(&self, ctype: ClientType, endpoint: &str) -> RequestBuilder {
+    /// - `visitor_data`: YouTube visitor data cookie
-        match ctype {
+    async fn request_builder(
+        &self,
+        ctype: ClientType,
+        endpoint: &str,
+        visitor_data: Option<&str>,
+    ) -> RequestBuilder {
+        let mut r = match ctype {
             ClientType::Desktop => self
                 .client
                 .inner
@@ -1215,7 +1221,7 @@ impl RustyPipeQuery {
                 .header("X-YouTube-Client-Name", "67")
                 .header(
                     "X-YouTube-Client-Version",
-                    self.client.get_music_client_version().await,
+                        self.client.get_music_client_version().await
                 ),
             ClientType::TvHtml5Embed => self
                 .client
@@ -1258,7 +1264,11 @@ impl RustyPipeQuery {
                     ),
                 )
                 .header("X-Goog-Api-Format-Version", "2"),
+        };
+        if let Some(vdata) = self.opts.visitor_data.as_deref().or(visitor_data) {
+            r = r.header("X-Goog-EOM-Visitor-Id", vdata);
         }
+        r
     }
 
     /// Get a YouTube visitor data cookie, which is necessary for certain requests
@@ -1273,6 +1283,7 @@ impl RustyPipeQuery {
         &self,
         request: &Request,
         id: &str,
+        visitor_data: Option<&str>,
         deobf: Option<&DeobfData>,
     ) -> Result<RequestResult<M>, Error> {
         let response = self
@@ -1306,7 +1317,7 @@ impl RustyPipeQuery {
                     id,
                     self.opts.lang,
                     deobf,
-                    self.opts.visitor_data.as_deref(),
+                    self.opts.visitor_data.as_deref().or(visitor_data),
                 ) {
                     Ok(mapres) => Ok(mapres),
                     Err(e) => Err(e.into()),
@@ -1324,11 +1335,14 @@ impl RustyPipeQuery {
         &self,
         request: &Request,
         id: &str,
+        visitor_data: Option<&str>,
         deobf: Option<&DeobfData>,
     ) -> Result<RequestResult<M>, Error> {
         let mut last_resp = None;
         for n in 0..=self.client.inner.n_http_retries {
-            let resp = self.yt_request_attempt::<R, M>(request, id, deobf).await?;
+            let resp = self
+                .yt_request_attempt::<R, M>(request, id, visitor_data, deobf)
+                .await?;
 
             let err = match &resp.res {
                 Ok(_) => return Ok(resp),
@@ -1369,7 +1383,9 @@ impl RustyPipeQuery {
     /// - `method`: HTTP method
     /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
     /// - `body`: Serializable request body to be sent in json format
+    /// - `visitor_data`: YouTube visitor data cookie
     /// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs).
+    #[allow(clippy::too_many_arguments)]
     async fn execute_request_deobf<
         R: DeserializeOwned + MapResponse<M> + Debug,
         M,
@@ -1381,17 +1397,20 @@ impl RustyPipeQuery {
         id: &str,
         endpoint: &str,
         body: &B,
+        visitor_data: Option<&str>,
         deobf: Option<&DeobfData>,
     ) -> Result<M, Error> {
         tracing::debug!("getting {}({})", operation, id);
 
         let request = self
-            .request_builder(ctype, endpoint)
+            .request_builder(ctype, endpoint, visitor_data)
             .await
             .json(body)
             .build()?;
 
-        let req_res = self.yt_request::<R, M>(&request, id, deobf).await?;
+        let req_res = self
+            .yt_request::<R, M>(&request, id, visitor_data, deobf)
+            .await?;
 
         // Uncomment to debug response text
         // println!("{}", &req_res.body);
@@ -1477,11 +1496,55 @@ impl RustyPipeQuery {
         endpoint: &str,
         body: &B,
     ) -> Result<M, Error> {
-        self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None)
+        self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None, None)
             .await
     }
 
+    /// Execute a request to the YouTube API, then map the response.
+    ///
+    /// Creates a report in case of failure for easy debugging.
+    ///
+    /// # Parameters
+    /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
+    /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`)
+    /// - `id`: ID of the requested entity (Video ID, Channel ID, ...).
+    ///   The ID is included in reports and is also passed to the mapper for validating the response.
+    ///   Set it to an empty string if you are not requesting an entity with an ID.
+    /// - `method`: HTTP method
+    /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
+    /// - `body`: Serializable request body to be sent in json format
+    /// - `visitor_data`: YouTube visitor data cookie
+    async fn execute_request_vdata<
+        R: DeserializeOwned + MapResponse<M> + Debug,
+        M,
+        B: Serialize + ?Sized,
+    >(
+        &self,
+        ctype: ClientType,
+        operation: &str,
+        id: &str,
+        endpoint: &str,
+        body: &B,
+        visitor_data: Option<&str>,
+    ) -> Result<M, Error> {
+        self.execute_request_deobf::<R, M, B>(
+            ctype,
+            operation,
+            id,
+            endpoint,
+            body,
+            visitor_data,
+            None,
+        )
+        .await
+    }
+
     /// Execute a request to the YouTube API and return the response string
+    ///
+    /// # Parameters
+    /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
+    /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
+    /// - `body`: Serializable request body to be sent in json format
     pub async fn raw<B: Serialize + ?Sized>(
         &self,
         ctype: ClientType,
@@ -1489,7 +1552,7 @@ impl RustyPipeQuery {
         body: &B,
     ) -> Result<String, Error> {
         let request = self
-            .request_builder(ctype, endpoint)
+            .request_builder(ctype, endpoint, None)
             .await
             .json(body)
             .build()?;
@@ -1519,13 +1582,13 @@ trait MapResponse<T> {
     ///   that the returned entity matches this ID and return an error instead.
     /// - `lang`: Language of the request. Used for mapping localized information like dates.
     /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method)
-    /// - `vdata`: Visitor data option of the client
+    /// - `visitor_data`: Visitor data option of the client
     fn map_response(
         self,
         id: &str,
         lang: Language,
         deobf: Option<&DeobfData>,
-        vdata: Option<&str>,
+        visitor_data: Option<&str>,
     ) -> Result<MapResult<T>, ExtractionError>;
 }
 

src/client/music_details.rs

@@ -4,7 +4,10 @@ use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem,
+    },
     param::Language,
     serializer::MapResult,
 };
@@ -132,12 +135,13 @@ impl RustyPipeQuery {
             tuner_setting_value: "AUTOMIX_SETTING_NORMAL",
         };
 
-        self.execute_request::<response::MusicDetails, _, _>(
+        self.execute_request_vdata::<response::MusicDetails, _, _>(
             ClientType::DesktopMusic,
             "music_radio",
             radio_id,
             "next",
             &request_body,
+            Some(&visitor_data),
         )
         .await
     }
@@ -293,13 +297,7 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
             .map(|c| c.next_continuation_data.continuation);
 
         Ok(MapResult {
-            c: Paginator::new_ext(
+            c: Paginator::new_ext(None, tracks, ctoken, None, ContinuationEndpoint::MusicNext),
-                None,
-                tracks,
-                ctoken,
-                None,
-                crate::model::paginator::ContinuationEndpoint::MusicNext,
-            ),
             warnings,
         })
     }

src/client/music_playlist.rs

@@ -2,7 +2,10 @@ use std::{borrow::Cow, fmt::Debug};
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem,
+    },
     serializer::MapResult,
     util::{self, TryRemove, DOT_SEPARATOR},
 };
@@ -23,18 +26,27 @@ impl RustyPipeQuery {
         playlist_id: S,
     ) -> Result<MusicPlaylist, Error> {
         let playlist_id = playlist_id.as_ref();
-        let context = self.get_context(ClientType::DesktopMusic, true, None).await;
+        // YTM playlists require visitor data for continuations to work
+        let visitor_data = if playlist_id.starts_with("RD") {
+            Some(self.get_visitor_data().await?)
+        } else {
+            None
+        };
+        let context = self
+            .get_context(ClientType::DesktopMusic, true, visitor_data.as_deref())
+            .await;
         let request_body = QBrowse {
             context,
             browse_id: &format!("VL{playlist_id}"),
         };
 
-        self.execute_request::<response::MusicPlaylist, _, _>(
+        self.execute_request_vdata::<response::MusicPlaylist, _, _>(
             ClientType::DesktopMusic,
             "music_playlist",
             playlist_id,
             "browse",
             &request_body,
+            visitor_data.as_deref(),
         )
         .await
     }
@@ -127,7 +139,7 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
         id: &str,
         lang: crate::param::Language,
         _deobf: Option<&crate::deobfuscate::DeobfData>,
-        _vdata: Option<&str>,
+        vdata: Option<&str>,
     ) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
         // dbg!(&self);
 
@@ -251,15 +263,15 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
                     track_count,
                     map_res.c,
                     ctoken,
-                    None,
+                    vdata.map(str::to_owned),
-                    crate::model::paginator::ContinuationEndpoint::MusicBrowse,
+                    ContinuationEndpoint::MusicBrowse,
                 ),
                 related_playlists: Paginator::new_ext(
                     None,
                     Vec::new(),
                     related_ctoken,
-                    None,
+                    vdata.map(str::to_owned),
-                    crate::model::paginator::ContinuationEndpoint::MusicBrowse,
+                    ContinuationEndpoint::MusicBrowse,
                 ),
             },
             warnings: map_res.warnings,

src/client/music_search.rs

@@ -6,8 +6,10 @@ use crate::{
     client::response::music_item::MusicListMapper,
     error::{Error, ExtractionError},
     model::{
-        paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem,
+        paginator::{ContinuationEndpoint, Paginator},
-        MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem,
+        traits::FromYtItem,
+        AlbumItem, ArtistItem, MusicPlaylistItem, MusicSearchFiltered, MusicSearchResult,
+        MusicSearchSuggestion, TrackItem,
     },
     serializer::MapResult,
 };
@@ -287,7 +289,7 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
         _id: &str,
         lang: crate::param::Language,
         _deobf: Option<&crate::deobfuscate::DeobfData>,
-        _vdata: Option<&str>,
+        vdata: Option<&str>,
     ) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> {
         // dbg!(&self);
 
@@ -332,8 +334,8 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
                     None,
                     map_res.c,
                     ctoken,
-                    None,
+                    vdata.map(str::to_owned),
-                    crate::model::paginator::ContinuationEndpoint::MusicSearch,
+                    ContinuationEndpoint::MusicSearch,
                 ),
                 corrected_query,
             },

src/client/pagination.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt::Debug;
 
 use crate::error::{Error, ExtractionError};
@@ -22,8 +23,13 @@ impl RustyPipeQuery {
     ) -> Result<Paginator<T>, Error> {
         let ctoken = ctoken.as_ref();
         if endpoint.is_music() {
+            // Visitor data is required for YTM continuations
+            let visitor_data = match visitor_data {
+                Some(vd) => Cow::Borrowed(vd),
+                None => Cow::Owned(self.get_visitor_data().await?),
+            };
             let context = self
-                .get_context(ClientType::DesktopMusic, true, visitor_data)
+                .get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
                 .await;
             let request_body = QContinuation {
                 context,
@@ -31,16 +37,17 @@ impl RustyPipeQuery {
             };
 
             let p = self
-                .execute_request::<response::MusicContinuation, Paginator<MusicItem>, _>(
+                .execute_request_vdata::<response::MusicContinuation, Paginator<MusicItem>, _>(
                     ClientType::DesktopMusic,
                     "music_continuation",
                     ctoken,
                     endpoint.as_str(),
                     &request_body,
+                    Some(&visitor_data),
                 )
                 .await?;
 
-            Ok(map_ytm_paginator(p, visitor_data, endpoint))
+            Ok(map_ytm_paginator(p, Some(&visitor_data), endpoint))
         } else {
             let context = self
                 .get_context(ClientType::Desktop, true, visitor_data)
@@ -211,6 +218,9 @@ impl<T: FromYtItem> Paginator<T> {
                 let mut items = paginator.items;
                 self.items.append(&mut items);
                 self.ctoken = paginator.ctoken;
+                if paginator.visitor_data.is_some() {
+                    self.visitor_data = paginator.visitor_data;
+                }
                 Ok(true)
             }
             Ok(None) => Ok(false),
@@ -285,6 +295,9 @@ macro_rules! paginator {
                         let mut items = paginator.items;
                         self.items.append(&mut items);
                         self.ctoken = paginator.ctoken;
+                        if paginator.visitor_data.is_some() {
+                            self.visitor_data = paginator.visitor_data;
+                        }
                         Ok(true)
                     }
                     Ok(None) => Ok(false),

src/client/player.rs

@@ -139,6 +139,7 @@ impl RustyPipeQuery {
             video_id,
             "player",
             &request_body,
+            None,
             Some(&deobf),
         )
         .await

src/client/search.rs

@@ -4,7 +4,10 @@ use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, SearchResult, YouTubeItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        SearchResult, YouTubeItem,
+    },
     param::search_filter::SearchFilter,
 };
 
@@ -119,7 +122,7 @@ impl MapResponse<SearchResult> for response::Search {
                     mapper.items,
                     mapper.ctoken,
                     None,
-                    crate::model::paginator::ContinuationEndpoint::Search,
+                    ContinuationEndpoint::Search,
                 ),
                 corrected_query: mapper.corrected_query,
                 visitor_data: self

src/client/trends.rs

@@ -2,7 +2,10 @@ use std::borrow::Cow;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, VideoItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        VideoItem,
+    },
     param::Language,
     serializer::MapResult,
 };
@@ -124,7 +127,7 @@ fn map_startpage_videos(
             mapper.items,
             mapper.ctoken,
             visitor_data,
-            crate::model::paginator::ContinuationEndpoint::Browse,
+            ContinuationEndpoint::Browse,
         ),
         warnings: mapper.warnings,
     }

src/client/video_details.rs

@@ -4,7 +4,10 @@ use serde::Serialize;
 
 use crate::{
     error::{Error, ExtractionError},
-    model::{paginator::Paginator, ChannelTag, Chapter, Comment, VideoDetails, VideoItem},
+    model::{
+        paginator::{ContinuationEndpoint, Paginator},
+        ChannelTag, Chapter, Comment, VideoDetails, VideoItem,
+    },
     param::Language,
     serializer::MapResult,
     util::{self, timeago, TryRemove},
@@ -360,14 +363,14 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
                     Vec::new(),
                     comment_ctoken,
                     visitor_data.clone(),
-                    crate::model::paginator::ContinuationEndpoint::Next,
+                    ContinuationEndpoint::Next,
                 ),
                 latest_comments: Paginator::new_ext(
                     comment_count,
                     Vec::new(),
                     latest_comments_ctoken,
                     visitor_data.clone(),
-                    crate::model::paginator::ContinuationEndpoint::Next,
+                    ContinuationEndpoint::Next,
                 ),
                 visitor_data,
             },
@@ -459,7 +462,7 @@ fn map_recommendations(
             mapper.items,
             mapper.ctoken,
             visitor_data,
-            crate::model::paginator::ContinuationEndpoint::Next,
+            ContinuationEndpoint::Next,
         ),
         warnings: mapper.warnings,
     }

tests/youtube.rs

@@ -1368,17 +1368,21 @@ fn music_playlist(
 }
 
 #[rstest]
-fn music_playlist_cont(rp: RustyPipe) {
+#[case::user("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj")]
-    let mut playlist = tokio_test::block_on(
+#[case::ytm("RDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM")]
-        rp.query()
+fn music_playlist_cont(#[case] id: &str, rp: RustyPipe) {
-            .music_playlist("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"),
+    let mut playlist = tokio_test::block_on(rp.query().music_playlist(id)).unwrap();
-    )
-    .unwrap();
 
-    tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), usize::MAX)).unwrap();
+    tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), 5)).unwrap();
 
-    assert_gte(playlist.tracks.items.len(), 100, "tracks");
+    let track_count = playlist.track_count.unwrap();
-    assert_gte(playlist.tracks.count.unwrap(), 100, "track count");
+    assert_gte(track_count, 100, "tracks");
+
+    assert_eq!(track_count, playlist.tracks.count.unwrap());
+    assert_eq!(
+        usize::try_from(track_count).unwrap(),
+        playlist.tracks.items.len()
+    );
 }
 
 #[rstest]