Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: send visitor data for YTM playlists

Browse source
This commit is contained in:
ThetaDev 2023-09-28 01:00:00 +02:00
parent 127596687b
commit b25e9ebbb7
11 changed files with 156 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/client/channel.rs
View file

@ -238,7 +238,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
mapper.items,
mapper.ctoken,
visitor_data,
crate::model::paginator::ContinuationEndpoint::Browse,
ContinuationEndpoint::Browse,
);
Ok(MapResult {

85
src/client/mod.rs
View file

@ -1185,8 +1185,14 @@ impl RustyPipeQuery {
/// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
/// - `method`: HTTP method
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
async fn request_builder(&self, ctype: ClientType, endpoint: &str) -> RequestBuilder {
match ctype {
/// - `visitor_data`: YouTube visitor data cookie
async fn request_builder(
&self,
ctype: ClientType,
endpoint: &str,
visitor_data: Option<&str>,
) -> RequestBuilder {
let mut r = match ctype {
ClientType::Desktop => self
.client
.inner
@ -1215,7 +1221,7 @@ impl RustyPipeQuery {
.header("X-YouTube-Client-Name", "67")
.header(
"X-YouTube-Client-Version",
self.client.get_music_client_version().await,
self.client.get_music_client_version().await
),
ClientType::TvHtml5Embed => self
.client
@ -1258,7 +1264,11 @@ impl RustyPipeQuery {
),
)
.header("X-Goog-Api-Format-Version", "2"),
};
if let Some(vdata) = self.opts.visitor_data.as_deref().or(visitor_data) {
r = r.header("X-Goog-EOM-Visitor-Id", vdata);
}
r
}
/// Get a YouTube visitor data cookie, which is necessary for certain requests
@ -1273,6 +1283,7 @@ impl RustyPipeQuery {
&self,
request: &Request,
id: &str,
visitor_data: Option<&str>,
deobf: Option<&DeobfData>,
) -> Result<RequestResult<M>, Error> {
let response = self
@ -1306,7 +1317,7 @@ impl RustyPipeQuery {
id,
self.opts.lang,
deobf,
self.opts.visitor_data.as_deref(),
self.opts.visitor_data.as_deref().or(visitor_data),
) {
Ok(mapres) => Ok(mapres),
Err(e) => Err(e.into()),
@ -1324,11 +1335,14 @@ impl RustyPipeQuery {
&self,
request: &Request,
id: &str,
visitor_data: Option<&str>,
deobf: Option<&DeobfData>,
) -> Result<RequestResult<M>, Error> {
let mut last_resp = None;
for n in 0..=self.client.inner.n_http_retries {
let resp = self.yt_request_attempt::<R, M>(request, id, deobf).await?;
let resp = self
.yt_request_attempt::<R, M>(request, id, visitor_data, deobf)
.await?;
let err = match &resp.res {
Ok(_) => return Ok(resp),
@ -1369,7 +1383,9 @@ impl RustyPipeQuery {
/// - `method`: HTTP method
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
/// - `body`: Serializable request body to be sent in json format
/// - `visitor_data`: YouTube visitor data cookie
/// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs).
#[allow(clippy::too_many_arguments)]
async fn execute_request_deobf<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
@ -1381,17 +1397,20 @@ impl RustyPipeQuery {
id: &str,
endpoint: &str,
body: &B,
visitor_data: Option<&str>,
deobf: Option<&DeobfData>,
) -> Result<M, Error> {
tracing::debug!("getting {}({})", operation, id);
let request = self
.request_builder(ctype, endpoint)
.request_builder(ctype, endpoint, visitor_data)
.await
.json(body)
.build()?;
let req_res = self.yt_request::<R, M>(&request, id, deobf).await?;
let req_res = self
.yt_request::<R, M>(&request, id, visitor_data, deobf)
.await?;
// Uncomment to debug response text
// println!("{}", &req_res.body);
@ -1477,11 +1496,55 @@ impl RustyPipeQuery {
endpoint: &str,
body: &B,
) -> Result<M, Error> {
self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None)
self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None, None)
.await
}
/// Execute a request to the YouTube API, then map the response.
///
/// Creates a report in case of failure for easy debugging.
///
/// # Parameters
/// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
/// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`)
/// - `id`: ID of the requested entity (Video ID, Channel ID, ...).
/// The ID is included in reports and is also passed to the mapper for validating the response.
/// Set it to an empty string if you are not requesting an entity with an ID.
/// - `method`: HTTP method
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
/// - `body`: Serializable request body to be sent in json format
/// - `visitor_data`: YouTube visitor data cookie
async fn execute_request_vdata<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
B: Serialize + ?Sized,
>(
&self,
ctype: ClientType,
operation: &str,
id: &str,
endpoint: &str,
body: &B,
visitor_data: Option<&str>,
) -> Result<M, Error> {
self.execute_request_deobf::<R, M, B>(
ctype,
operation,
id,
endpoint,
body,
visitor_data,
None,
)
.await
}
/// Execute a request to the YouTube API and return the response string
///
/// # Parameters
/// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
/// - `body`: Serializable request body to be sent in json format
pub async fn raw<B: Serialize + ?Sized>(
&self,
ctype: ClientType,
@ -1489,7 +1552,7 @@ impl RustyPipeQuery {
body: &B,
) -> Result<String, Error> {
let request = self
.request_builder(ctype, endpoint)
.request_builder(ctype, endpoint, None)
.await
.json(body)
.build()?;
@ -1519,13 +1582,13 @@ trait MapResponse<T> {
/// that the returned entity matches this ID and return an error instead.
/// - `lang`: Language of the request. Used for mapping localized information like dates.
/// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method)
/// - `vdata`: Visitor data option of the client
/// - `visitor_data`: Visitor data option of the client
fn map_response(
self,
id: &str,
lang: Language,
deobf: Option<&DeobfData>,
vdata: Option<&str>,
visitor_data: Option<&str>,
) -> Result<MapResult<T>, ExtractionError>;
}

16
src/client/music_details.rs
View file

@ -4,7 +4,10 @@ use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem},
model::{
paginator::{ContinuationEndpoint, Paginator},
ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem,
},
param::Language,
serializer::MapResult,
};
@ -132,12 +135,13 @@ impl RustyPipeQuery {
tuner_setting_value: "AUTOMIX_SETTING_NORMAL",
};
self.execute_request::<response::MusicDetails, _, _>(
self.execute_request_vdata::<response::MusicDetails, _, _>(
ClientType::DesktopMusic,
"music_radio",
radio_id,
"next",
&request_body,
Some(&visitor_data),
)
.await
}
@ -293,13 +297,7 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
.map(|c| c.next_continuation_data.continuation);
Ok(MapResult {
c: Paginator::new_ext(
None,
tracks,
ctoken,
None,
crate::model::paginator::ContinuationEndpoint::MusicNext,
),
c: Paginator::new_ext(None, tracks, ctoken, None, ContinuationEndpoint::MusicNext),
warnings,
})
}

28
src/client/music_playlist.rs
View file

@ -2,7 +2,10 @@ use std::{borrow::Cow, fmt::Debug};
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem},
model::{
paginator::{ContinuationEndpoint, Paginator},
AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem,
},
serializer::MapResult,
util::{self, TryRemove, DOT_SEPARATOR},
};
@ -23,18 +26,27 @@ impl RustyPipeQuery {
playlist_id: S,
) -> Result<MusicPlaylist, Error> {
let playlist_id = playlist_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
// YTM playlists require visitor data for continuations to work
let visitor_data = if playlist_id.starts_with("RD") {
Some(self.get_visitor_data().await?)
} else {
None
};
let context = self
.get_context(ClientType::DesktopMusic, true, visitor_data.as_deref())
.await;
let request_body = QBrowse {
context,
browse_id: &format!("VL{playlist_id}"),
};
self.execute_request::<response::MusicPlaylist, _, _>(
self.execute_request_vdata::<response::MusicPlaylist, _, _>(
ClientType::DesktopMusic,
"music_playlist",
playlist_id,
"browse",
&request_body,
visitor_data.as_deref(),
)
.await
}
@ -127,7 +139,7 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
vdata: Option<&str>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
// dbg!(&self);
@ -251,15 +263,15 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
track_count,
map_res.c,
ctoken,
None,
crate::model::paginator::ContinuationEndpoint::MusicBrowse,
vdata.map(str::to_owned),
ContinuationEndpoint::MusicBrowse,
),
related_playlists: Paginator::new_ext(
None,
Vec::new(),
related_ctoken,
None,
crate::model::paginator::ContinuationEndpoint::MusicBrowse,
vdata.map(str::to_owned),
ContinuationEndpoint::MusicBrowse,
),
},
warnings: map_res.warnings,

12
src/client/music_search.rs
View file

@ -6,8 +6,10 @@ use crate::{
client::response::music_item::MusicListMapper,
error::{Error, ExtractionError},
model::{
paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem,
MusicSearchFiltered, MusicSearchResult, MusicSearchSuggestion, TrackItem,
paginator::{ContinuationEndpoint, Paginator},
traits::FromYtItem,
AlbumItem, ArtistItem, MusicPlaylistItem, MusicSearchFiltered, MusicSearchResult,
MusicSearchSuggestion, TrackItem,
},
serializer::MapResult,
};
@ -287,7 +289,7 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
vdata: Option<&str>,
) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> {
// dbg!(&self);
@ -332,8 +334,8 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
None,
map_res.c,
ctoken,
None,
crate::model::paginator::ContinuationEndpoint::MusicSearch,
vdata.map(str::to_owned),
ContinuationEndpoint::MusicSearch,
),
corrected_query,
},

19
src/client/pagination.rs
View file

@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::fmt::Debug;
use crate::error::{Error, ExtractionError};
@ -22,8 +23,13 @@ impl RustyPipeQuery {
) -> Result<Paginator<T>, Error> {
let ctoken = ctoken.as_ref();
if endpoint.is_music() {
// Visitor data is required for YTM continuations
let visitor_data = match visitor_data {
Some(vd) => Cow::Borrowed(vd),
None => Cow::Owned(self.get_visitor_data().await?),
};
let context = self
.get_context(ClientType::DesktopMusic, true, visitor_data)
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QContinuation {
context,
@ -31,16 +37,17 @@ impl RustyPipeQuery {
};
let p = self
.execute_request::<response::MusicContinuation, Paginator<MusicItem>, _>(
.execute_request_vdata::<response::MusicContinuation, Paginator<MusicItem>, _>(
ClientType::DesktopMusic,
"music_continuation",
ctoken,
endpoint.as_str(),
&request_body,
Some(&visitor_data),
)
.await?;
Ok(map_ytm_paginator(p, visitor_data, endpoint))
Ok(map_ytm_paginator(p, Some(&visitor_data), endpoint))
} else {
let context = self
.get_context(ClientType::Desktop, true, visitor_data)
@ -211,6 +218,9 @@ impl<T: FromYtItem> Paginator<T> {
let mut items = paginator.items;
self.items.append(&mut items);
self.ctoken = paginator.ctoken;
if paginator.visitor_data.is_some() {
self.visitor_data = paginator.visitor_data;
}
Ok(true)
}
Ok(None) => Ok(false),
@ -285,6 +295,9 @@ macro_rules! paginator {
let mut items = paginator.items;
self.items.append(&mut items);
self.ctoken = paginator.ctoken;
if paginator.visitor_data.is_some() {
self.visitor_data = paginator.visitor_data;
}
Ok(true)
}
Ok(None) => Ok(false),

1
src/client/player.rs
View file

@ -139,6 +139,7 @@ impl RustyPipeQuery {
video_id,
"player",
&request_body,
None,
Some(&deobf),
)
.await

7
src/client/search.rs
View file

@ -4,7 +4,10 @@ use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, SearchResult, YouTubeItem},
model::{
paginator::{ContinuationEndpoint, Paginator},
SearchResult, YouTubeItem,
},
param::search_filter::SearchFilter,
};
@ -119,7 +122,7 @@ impl MapResponse<SearchResult> for response::Search {
mapper.items,
mapper.ctoken,
None,
crate::model::paginator::ContinuationEndpoint::Search,
ContinuationEndpoint::Search,
),
corrected_query: mapper.corrected_query,
visitor_data: self

7
src/client/trends.rs
View file

@ -2,7 +2,10 @@ use std::borrow::Cow;
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, VideoItem},
model::{
paginator::{ContinuationEndpoint, Paginator},
VideoItem,
},
param::Language,
serializer::MapResult,
};
@ -124,7 +127,7 @@ fn map_startpage_videos(
mapper.items,
mapper.ctoken,
visitor_data,
crate::model::paginator::ContinuationEndpoint::Browse,
ContinuationEndpoint::Browse,
),
warnings: mapper.warnings,
}

11
src/client/video_details.rs
View file

@ -4,7 +4,10 @@ use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, ChannelTag, Chapter, Comment, VideoDetails, VideoItem},
model::{
paginator::{ContinuationEndpoint, Paginator},
ChannelTag, Chapter, Comment, VideoDetails, VideoItem,
},
param::Language,
serializer::MapResult,
util::{self, timeago, TryRemove},
@ -360,14 +363,14 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
Vec::new(),
comment_ctoken,
visitor_data.clone(),
crate::model::paginator::ContinuationEndpoint::Next,
ContinuationEndpoint::Next,
),
latest_comments: Paginator::new_ext(
comment_count,
Vec::new(),
latest_comments_ctoken,
visitor_data.clone(),
crate::model::paginator::ContinuationEndpoint::Next,
ContinuationEndpoint::Next,
),
visitor_data,
},
@ -459,7 +462,7 @@ fn map_recommendations(
mapper.items,
mapper.ctoken,
visitor_data,
crate::model::paginator::ContinuationEndpoint::Next,
ContinuationEndpoint::Next,
),
warnings: mapper.warnings,
}

22
tests/youtube.rs
View file

@ -1368,17 +1368,21 @@ fn music_playlist(
}
#[rstest]
fn music_playlist_cont(rp: RustyPipe) {
let mut playlist = tokio_test::block_on(
rp.query()
.music_playlist("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj"),
)
.unwrap();
#[case::user("PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj")]
#[case::ytm("RDCLAK5uy_kb7EBi6y3GrtJri4_ZH56Ms786DFEimbM")]
fn music_playlist_cont(#[case] id: &str, rp: RustyPipe) {
let mut playlist = tokio_test::block_on(rp.query().music_playlist(id)).unwrap();
tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), usize::MAX)).unwrap();
tokio_test::block_on(playlist.tracks.extend_pages(rp.query(), 5)).unwrap();
assert_gte(playlist.tracks.items.len(), 100, "tracks");
assert_gte(playlist.tracks.count.unwrap(), 100, "track count");
let track_count = playlist.track_count.unwrap();
assert_gte(track_count, 100, "tracks");
assert_eq!(track_count, playlist.tracks.count.unwrap());
assert_eq!(
usize::try_from(track_count).unwrap(),
playlist.tracks.items.len()
);
}
#[rstest]