diff --git a/src/client/channel.rs b/src/client/channel.rs index a002908..3116348 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -238,7 +238,7 @@ impl MapResponse>> for response::Channel { mapper.items, mapper.ctoken, visitor_data, - crate::model::paginator::ContinuationEndpoint::Browse, + ContinuationEndpoint::Browse, ); Ok(MapResult { diff --git a/src/client/mod.rs b/src/client/mod.rs index 6c688de..5b21882 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1185,8 +1185,14 @@ impl RustyPipeQuery { /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) /// - `method`: HTTP method /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - async fn request_builder(&self, ctype: ClientType, endpoint: &str) -> RequestBuilder { - match ctype { + /// - `visitor_data`: YouTube visitor data cookie + async fn request_builder( + &self, + ctype: ClientType, + endpoint: &str, + visitor_data: Option<&str>, + ) -> RequestBuilder { + let mut r = match ctype { ClientType::Desktop => self .client .inner @@ -1215,7 +1221,7 @@ impl RustyPipeQuery { .header("X-YouTube-Client-Name", "67") .header( "X-YouTube-Client-Version", - self.client.get_music_client_version().await, + self.client.get_music_client_version().await ), ClientType::TvHtml5Embed => self .client @@ -1258,7 +1264,11 @@ impl RustyPipeQuery { ), ) .header("X-Goog-Api-Format-Version", "2"), + }; + if let Some(vdata) = self.opts.visitor_data.as_deref().or(visitor_data) { + r = r.header("X-Goog-EOM-Visitor-Id", vdata); } + r } /// Get a YouTube visitor data cookie, which is necessary for certain requests @@ -1273,6 +1283,7 @@ impl RustyPipeQuery { &self, request: &Request, id: &str, + visitor_data: Option<&str>, deobf: Option<&DeobfData>, ) -> Result, Error> { let response = self @@ -1306,7 +1317,7 @@ impl RustyPipeQuery { id, self.opts.lang, deobf, - self.opts.visitor_data.as_deref(), + self.opts.visitor_data.as_deref().or(visitor_data), ) { Ok(mapres) => Ok(mapres), Err(e) => Err(e.into()), @@ -1324,11 +1335,14 @@ impl RustyPipeQuery { &self, request: &Request, id: &str, + visitor_data: Option<&str>, deobf: Option<&DeobfData>, ) -> Result, Error> { let mut last_resp = None; for n in 0..=self.client.inner.n_http_retries { - let resp = self.yt_request_attempt::(request, id, deobf).await?; + let resp = self + .yt_request_attempt::(request, id, visitor_data, deobf) + .await?; let err = match &resp.res { Ok(_) => return Ok(resp), @@ -1369,7 +1383,9 @@ impl RustyPipeQuery { /// - `method`: HTTP method /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) /// - `body`: Serializable request body to be sent in json format + /// - `visitor_data`: YouTube visitor data cookie /// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs). + #[allow(clippy::too_many_arguments)] async fn execute_request_deobf< R: DeserializeOwned + MapResponse + Debug, M, @@ -1381,17 +1397,20 @@ impl RustyPipeQuery { id: &str, endpoint: &str, body: &B, + visitor_data: Option<&str>, deobf: Option<&DeobfData>, ) -> Result { tracing::debug!("getting {}({})", operation, id); let request = self - .request_builder(ctype, endpoint) + .request_builder(ctype, endpoint, visitor_data) .await .json(body) .build()?; - let req_res = self.yt_request::(&request, id, deobf).await?; + let req_res = self + .yt_request::(&request, id, visitor_data, deobf) + .await?; // Uncomment to debug response text // println!("{}", &req_res.body); @@ -1477,11 +1496,55 @@ impl RustyPipeQuery { endpoint: &str, body: &B, ) -> Result { - self.execute_request_deobf::(ctype, operation, id, endpoint, body, None) + self.execute_request_deobf::(ctype, operation, id, endpoint, body, None, None) .await } + /// Execute a request to the YouTube API, then map the response. + /// + /// Creates a report in case of failure for easy debugging. + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) + /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). + /// The ID is included in reports and is also passed to the mapper for validating the response. + /// Set it to an empty string if you are not requesting an entity with an ID. + /// - `method`: HTTP method + /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) + /// - `body`: Serializable request body to be sent in json format + /// - `visitor_data`: YouTube visitor data cookie + async fn execute_request_vdata< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + id: &str, + endpoint: &str, + body: &B, + visitor_data: Option<&str>, + ) -> Result { + self.execute_request_deobf::( + ctype, + operation, + id, + endpoint, + body, + visitor_data, + None, + ) + .await + } + /// Execute a request to the YouTube API and return the response string + /// + /// # Parameters + /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) + /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) + /// - `body`: Serializable request body to be sent in json format pub async fn raw( &self, ctype: ClientType, @@ -1489,7 +1552,7 @@ impl RustyPipeQuery { body: &B, ) -> Result { let request = self - .request_builder(ctype, endpoint) + .request_builder(ctype, endpoint, None) .await .json(body) .build()?; @@ -1519,13 +1582,13 @@ trait MapResponse { /// that the returned entity matches this ID and return an error instead. /// - `lang`: Language of the request. Used for mapping localized information like dates. /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) - /// - `vdata`: Visitor data option of the client + /// - `visitor_data`: Visitor data option of the client fn map_response( self, id: &str, lang: Language, deobf: Option<&DeobfData>, - vdata: Option<&str>, + visitor_data: Option<&str>, ) -> Result, ExtractionError>; } diff --git a/src/client/music_details.rs b/src/client/music_details.rs index 60980c9..0e189ab 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -4,7 +4,10 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem}, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem, + }, param::Language, serializer::MapResult, }; @@ -132,12 +135,13 @@ impl RustyPipeQuery { tuner_setting_value: "AUTOMIX_SETTING_NORMAL", }; - self.execute_request::( + self.execute_request_vdata::( ClientType::DesktopMusic, "music_radio", radio_id, "next", &request_body, + Some(&visitor_data), ) .await } @@ -293,13 +297,7 @@ impl MapResponse> for response::MusicDetails { .map(|c| c.next_continuation_data.continuation); Ok(MapResult { - c: Paginator::new_ext( - None, - tracks, - ctoken, - None, - crate::model::paginator::ContinuationEndpoint::MusicNext, - ), + c: Paginator::new_ext(None, tracks, ctoken, None, ContinuationEndpoint::MusicNext), warnings, }) } diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 00230e6..ecf76dc 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -2,7 +2,10 @@ use std::{borrow::Cow, fmt::Debug}; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem}, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem, + }, serializer::MapResult, util::{self, TryRemove, DOT_SEPARATOR}, }; @@ -23,18 +26,27 @@ impl RustyPipeQuery { playlist_id: S, ) -> Result { let playlist_id = playlist_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; + // YTM playlists require visitor data for continuations to work + let visitor_data = if playlist_id.starts_with("RD") { + Some(self.get_visitor_data().await?) + } else { + None + }; + let context = self + .get_context(ClientType::DesktopMusic, true, visitor_data.as_deref()) + .await; let request_body = QBrowse { context, browse_id: &format!("VL{playlist_id}"), }; - self.execute_request::( + self.execute_request_vdata::( ClientType::DesktopMusic, "music_playlist", playlist_id, "browse", &request_body, + visitor_data.as_deref(), ) .await } @@ -127,7 +139,7 @@ impl MapResponse for response::MusicPlaylist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, - _vdata: Option<&str>, + vdata: Option<&str>, ) -> Result, ExtractionError> { // dbg!(&self); @@ -251,15 +263,15 @@ impl MapResponse for response::MusicPlaylist { track_count, map_res.c, ctoken, - None, - crate::model::paginator::ContinuationEndpoint::MusicBrowse, + vdata.map(str::to_owned), + ContinuationEndpoint::MusicBrowse, ), related_playlists: Paginator::new_ext( None, Vec::new(), related_ctoken, - None, - crate::model::paginator::ContinuationEndpoint::MusicBrowse, + vdata.map(str::to_owned), + ContinuationEndpoint::MusicBrowse, ), }, warnings: map_res.warnings, diff --git a/src/client/music_search.rs b/src/client/music_search.rs index 765f913..309bc1e 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -6,8 +6,10 @@ use crate::{ client::response::music_item::MusicListMapper, error::{Error, ExtractionError}, model::{ - paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem, - MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem, + paginator::{ContinuationEndpoint, Paginator}, + traits::FromYtItem, + AlbumItem, ArtistItem, MusicPlaylistItem, MusicSearchFiltered, MusicSearchResult, + MusicSearchSuggestion, TrackItem, }, serializer::MapResult, }; @@ -287,7 +289,7 @@ impl MapResponse> for response::MusicSearc _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, - _vdata: Option<&str>, + vdata: Option<&str>, ) -> Result>, ExtractionError> { // dbg!(&self); @@ -332,8 +334,8 @@ impl MapResponse> for response::MusicSearc None, map_res.c, ctoken, - None, - crate::model::paginator::ContinuationEndpoint::MusicSearch, + vdata.map(str::to_owned), + ContinuationEndpoint::MusicSearch, ), corrected_query, }, diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 4602b78..20ce792 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::fmt::Debug; use crate::error::{Error, ExtractionError}; @@ -22,8 +23,13 @@ impl RustyPipeQuery { ) -> Result, Error> { let ctoken = ctoken.as_ref(); if endpoint.is_music() { + // Visitor data is required for YTM continuations + let visitor_data = match visitor_data { + Some(vd) => Cow::Borrowed(vd), + None => Cow::Owned(self.get_visitor_data().await?), + }; let context = self - .get_context(ClientType::DesktopMusic, true, visitor_data) + .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) .await; let request_body = QContinuation { context, @@ -31,16 +37,17 @@ impl RustyPipeQuery { }; let p = self - .execute_request::, _>( + .execute_request_vdata::, _>( ClientType::DesktopMusic, "music_continuation", ctoken, endpoint.as_str(), &request_body, + Some(&visitor_data), ) .await?; - Ok(map_ytm_paginator(p, visitor_data, endpoint)) + Ok(map_ytm_paginator(p, Some(&visitor_data), endpoint)) } else { let context = self .get_context(ClientType::Desktop, true, visitor_data) @@ -211,6 +218,9 @@ impl Paginator { let mut items = paginator.items; self.items.append(&mut items); self.ctoken = paginator.ctoken; + if paginator.visitor_data.is_some() { + self.visitor_data = paginator.visitor_data; + } Ok(true) } Ok(None) => Ok(false), @@ -285,6 +295,9 @@ macro_rules! paginator { let mut items = paginator.items; self.items.append(&mut items); self.ctoken = paginator.ctoken; + if paginator.visitor_data.is_some() { + self.visitor_data = paginator.visitor_data; + } Ok(true) } Ok(None) => Ok(false), diff --git a/src/client/player.rs b/src/client/player.rs index c12b802..b948eda 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -139,6 +139,7 @@ impl RustyPipeQuery { video_id, "player", &request_body, + None, Some(&deobf), ) .await diff --git a/src/client/search.rs b/src/client/search.rs index 48da18f..a2e6001 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -4,7 +4,10 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, SearchResult, YouTubeItem}, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + SearchResult, YouTubeItem, + }, param::search_filter::SearchFilter, }; @@ -119,7 +122,7 @@ impl MapResponse for response::Search { mapper.items, mapper.ctoken, None, - crate::model::paginator::ContinuationEndpoint::Search, + ContinuationEndpoint::Search, ), corrected_query: mapper.corrected_query, visitor_data: self diff --git a/src/client/trends.rs b/src/client/trends.rs index 10bbbdc..8b6f3b8 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -2,7 +2,10 @@ use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, VideoItem}, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + VideoItem, + }, param::Language, serializer::MapResult, }; @@ -124,7 +127,7 @@ fn map_startpage_videos( mapper.items, mapper.ctoken, visitor_data, - crate::model::paginator::ContinuationEndpoint::Browse, + ContinuationEndpoint::Browse, ), warnings: mapper.warnings, } diff --git a/src/client/video_details.rs b/src/client/video_details.rs index aebe99f..32b63f4 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -4,7 +4,10 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, ChannelTag, Chapter, Comment, VideoDetails, VideoItem}, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + ChannelTag, Chapter, Comment, VideoDetails, VideoItem, + }, param::Language, serializer::MapResult, util::{self, timeago, TryRemove}, @@ -360,14 +363,14 @@ impl MapResponse for response::VideoDetails { Vec::new(), comment_ctoken, visitor_data.clone(), - crate::model::paginator::ContinuationEndpoint::Next, + ContinuationEndpoint::Next, ), latest_comments: Paginator::new_ext( comment_count, Vec::new(), latest_comments_ctoken, visitor_data.clone(), - crate::model::paginator::ContinuationEndpoint::Next, + ContinuationEndpoint::Next, ), visitor_data, }, @@ -459,7 +462,7 @@ fn map_recommendations( mapper.items, mapper.ctoken, visitor_data, - crate::model::paginator::ContinuationEndpoint::Next, + ContinuationEndpoint::Next, ), warnings: mapper.warnings, } diff --git a/tests/youtube.rs b/tests/youtube.rs index 918b51c..1054a61 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1368,17 +1368,21 @@ fn music_playlist( } #[rstest] -fn music_playlist_cont(rp: RustyPipe) { - let mut playlist = tokio_test::block_on( - rp.query() - .music_playlist("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"), - ) - .unwrap(); +#[case::user("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj")] +#[case::ytm("RDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM")] +fn music_playlist_cont(#[case] id: &str, rp: RustyPipe) { + let mut playlist = tokio_test::block_on(rp.query().music_playlist(id)).unwrap(); - tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), usize::MAX)).unwrap(); + tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), 5)).unwrap(); - assert_gte(playlist.tracks.items.len(), 100, "tracks"); - assert_gte(playlist.tracks.count.unwrap(), 100, "track count"); + let track_count = playlist.track_count.unwrap(); + assert_gte(track_count, 100, "tracks"); + + assert_eq!(track_count, playlist.tracks.count.unwrap()); + assert_eq!( + usize::try_from(track_count).unwrap(), + playlist.tracks.items.len() + ); } #[rstest]