From 7c4f44d09c4d813efff9e7d1059ddacd226b9e9d Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 23 Oct 2024 01:51:16 +0200 Subject: [PATCH] feat!: generate random visitorData, remove `RustyPipeQuery::get_context` and `YTContext<'a>` from public API --- codegen/src/abtest.rs | 25 +------ codegen/src/collect_album_types.rs | 4 - codegen/src/collect_large_numbers.rs | 5 -- codegen/src/collect_video_durations.rs | 1 - codegen/src/model.rs | 4 +- src/client/channel.rs | 15 +--- src/client/mod.rs | 100 +++++++++---------------- src/client/music_artist.rs | 16 +--- src/client/music_charts.rs | 5 +- src/client/music_details.rs | 18 +---- src/client/music_genres.rs | 4 - src/client/music_new.rs | 4 - src/client/music_playlist.rs | 20 +---- src/client/music_search.rs | 12 +-- src/client/pagination.rs | 43 +++++------ src/client/player.rs | 12 +-- src/client/playlist.rs | 21 +----- src/client/search.rs | 7 +- src/client/trends.rs | 2 - src/client/url_resolver.rs | 7 +- src/client/video_details.rs | 9 +-- src/util/mod.rs | 23 ++++++ 22 files changed, 99 insertions(+), 258 deletions(-) diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index eeb95d8..153a080 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -6,7 +6,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use num_enum::TryFromPrimitive; use once_cell::sync::Lazy; use regex::Regex; -use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext}; +use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery}; use rustypipe::model::{MusicItem, YouTubeItem}; use rustypipe::param::search_filter::{ItemType, SearchFilter}; use rustypipe::param::ChannelVideoTab; @@ -57,7 +57,6 @@ pub struct ABTestRes { #[derive(Debug, Serialize)] struct QVideo<'a> { - context: YTContext<'a>, video_id: &'a str, content_check_ok: bool, racy_check_ok: bool, @@ -66,7 +65,6 @@ struct QVideo<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowse<'a> { - context: YTContext<'a>, browse_id: &'a str, #[serde(skip_serializing_if = "Option::is_none")] params: Option<&'a str>, @@ -153,9 +151,7 @@ pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec { } pub async fn attributed_text_description(rp: &RustyPipeQuery) -> Result { - let context = rp.get_context(ClientType::Desktop, true, None).await; let q = QVideo { - context, video_id: "ZeerrnuLi5E", content_check_ok: false, racy_check_ok: false, @@ -190,13 +186,11 @@ pub async fn channel_handles_in_search_results(rp: &RustyPipeQuery) -> Result Result { - let context = rp.get_context(ClientType::Desktop, true, None).await; let res = rp .raw( ClientType::Desktop, "browse", &QBrowse { - context, browse_id: "FEtrending", params: None, }, @@ -207,13 +201,11 @@ pub async fn trends_video_tab(rp: &RustyPipeQuery) -> Result { } pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result { - let context = rp.get_context(ClientType::Desktop, true, None).await; let res = rp .raw( ClientType::Desktop, "browse", &QBrowse { - context, browse_id: "FEtrending", params: None, }, @@ -237,7 +229,6 @@ pub async fn discography_page(rp: &RustyPipeQuery) -> Result { ClientType::DesktopMusic, "browse", &QBrowse { - context: rp.get_context(ClientType::DesktopMusic, true, None).await, browse_id: id, params: None, }, @@ -301,7 +292,6 @@ pub async fn channel_about_modal(rp: &RustyPipeQuery) -> Result { ClientType::Desktop, "browse", &QBrowse { - context: rp.get_context(ClientType::Desktop, true, None).await, browse_id: id, params: None, }, @@ -317,7 +307,6 @@ pub async fn like_button_viewmodel(rp: &RustyPipeQuery) -> Result { ClientType::Desktop, "next", &QVideo { - context: rp.get_context(ClientType::Desktop, true, None).await, video_id: "ZeerrnuLi5E", content_check_ok: true, racy_check_ok: true, @@ -341,7 +330,6 @@ pub async fn music_playlist_two_column(rp: &RustyPipeQuery) -> Result { ClientType::DesktopMusic, "browse", &QBrowse { - context: rp.get_context(ClientType::DesktopMusic, true, None).await, browse_id: id, params: None, }, @@ -355,14 +343,7 @@ pub async fn comments_framework_update(rp: &RustyPipeQuery) -> Result { let continuation = "Eg0SC3dMZHBSN2d1S3k4GAYyJSIRIgt3TGRwUjdndUt5ODAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D"; let res = rp - .raw( - ClientType::Desktop, - "next", - &QCont { - context: rp.get_context(ClientType::Desktop, true, None).await, - continuation, - }, - ) + .raw(ClientType::Desktop, "next", &QCont { continuation }) .await .unwrap(); Ok(res.contains("\"frameworkUpdates\"")) @@ -375,7 +356,6 @@ pub async fn channel_shorts_lockup(rp: &RustyPipeQuery) -> Result { ClientType::Desktop, "browse", &QBrowse { - context: rp.get_context(ClientType::Desktop, true, None).await, browse_id: id, params: Some("EgZzaG9ydHPyBgUKA5oBAA%3D%3D"), }, @@ -392,7 +372,6 @@ pub async fn playlist_page_header_renderer(rp: &RustyPipeQuery) -> Result ClientType::Desktop, "browse", &QBrowse { - context: rp.get_context(ClientType::Desktop, true, None).await, browse_id: id, params: None, }, diff --git a/codegen/src/collect_album_types.rs b/codegen/src/collect_album_types.rs index 49fb56a..965f37d 100644 --- a/codegen/src/collect_album_types.rs +++ b/codegen/src/collect_album_types.rs @@ -95,11 +95,7 @@ struct HeaderRenderer { } async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String { - let context = query - .get_context(ClientType::DesktopMusic, true, None) - .await; let body = QBrowse { - context, browse_id: id, params: None, }; diff --git a/codegen/src/collect_large_numbers.rs b/codegen/src/collect_large_numbers.rs index 59979ef..e9ebbb7 100644 --- a/codegen/src/collect_large_numbers.rs +++ b/codegen/src/collect_large_numbers.rs @@ -350,7 +350,6 @@ async fn get_channel(query: &RustyPipeQuery, channel_id: &str) -> Result Result ClientType::DesktopMusic, "browse", &QBrowse { - context: query - .get_context(ClientType::DesktopMusic, true, None) - .await, browse_id: channel_id, params: None, }, diff --git a/codegen/src/collect_video_durations.rs b/codegen/src/collect_video_durations.rs index 8d5024f..fd097e9 100644 --- a/codegen/src/collect_video_durations.rs +++ b/codegen/src/collect_video_durations.rs @@ -270,7 +270,6 @@ async fn get_channel_vlengths( ClientType::Desktop, "browse", &QBrowse { - context: query.get_context(ClientType::Desktop, true, None).await, browse_id: channel_id, params: Some("EgZ2aWRlb3MYASAAMAE"), }, diff --git a/codegen/src/model.rs b/codegen/src/model.rs index c487d94..e14b913 100644 --- a/codegen/src/model.rs +++ b/codegen/src/model.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use ordered_hash_map::OrderedHashMap; -use rustypipe::{client::YTContext, model::AlbumType, param::Language}; +use rustypipe::{model::AlbumType, param::Language}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DefaultOnError, VecSkipError}; @@ -116,7 +116,6 @@ pub enum ExtItemType { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct QBrowse<'a> { - pub context: YTContext<'a>, pub browse_id: &'a str, #[serde(skip_serializing_if = "Option::is_none")] pub params: Option<&'a str>, @@ -125,7 +124,6 @@ pub struct QBrowse<'a> { #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct QCont<'a> { - pub context: YTContext<'a>, pub continuation: &'a str, } diff --git a/src/client/channel.rs b/src/client/channel.rs index 93bd602..9584697 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -16,14 +16,11 @@ use crate::{ util::{self, timeago, ProtoBuilder}, }; -use super::{ - response, ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery, YTContext, -}; +use super::{response, ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QChannel<'a> { - context: YTContext<'a>, browse_id: &'a str, params: ChannelTab, #[serde(skip_serializing_if = "Option::is_none")] @@ -63,9 +60,7 @@ impl RustyPipeQuery { operation: &str, ) -> Result>, Error> { let channel_id = channel_id.as_ref(); - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { - context, browse_id: channel_id, params, query, @@ -125,12 +120,10 @@ impl RustyPipeQuery { tab: ChannelVideoTab, order: ChannelOrder, ) -> Result, Error> { - let visitor_data = Some(self.get_visitor_data().await?); - self.continuation( order_ctoken(channel_id.as_ref(), tab, order, &random_target()), ContinuationEndpoint::Browse, - visitor_data.as_deref(), + None, ) .await } @@ -158,9 +151,7 @@ impl RustyPipeQuery { channel_id: S, ) -> Result>, Error> { let channel_id = channel_id.as_ref(); - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { - context, browse_id: channel_id, params: ChannelTab::Playlists, query: None, @@ -183,9 +174,7 @@ impl RustyPipeQuery { channel_id: S, ) -> Result { let channel_id = channel_id.as_ref(); - let context = self.get_context(ClientType::Desktop, false, None).await; let request_body = QContinuation { - context, continuation: &channel_info_ctoken(channel_id, &random_target()), }; diff --git a/src/client/mod.rs b/src/client/mod.rs index cc4aa7b..a1a707b 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -96,7 +96,7 @@ impl ClientType { /// YouTube context request parameter #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] -pub struct YTContext<'a> { +struct YTContext<'a> { client: ClientInfo<'a>, /// only used on desktop #[serde(skip_serializing_if = "Option::is_none")] @@ -119,8 +119,7 @@ struct ClientInfo<'a> { platform: &'a str, #[serde(skip_serializing_if = "Option::is_none")] original_url: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - visitor_data: Option<&'a str>, + visitor_data: &'a str, hl: Language, gl: Country, time_zone: &'a str, @@ -136,7 +135,7 @@ impl Default for ClientInfo<'_> { device_model: None, platform: "", original_url: None, - visitor_data: None, + visitor_data: "", hl: Language::En, gl: Country::Us, time_zone: "UTC", @@ -173,17 +172,22 @@ struct ThirdParty<'a> { embed_url: &'a str, } +#[derive(Debug, Serialize)] +struct QBody<'a, T> { + context: YTContext<'a>, + #[serde(flatten)] + body: T, +} + #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowse<'a> { - context: YTContext<'a>, browse_id: &'a str, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QBrowseParams<'a> { - context: YTContext<'a>, browse_id: &'a str, params: &'a str, } @@ -191,7 +195,6 @@ struct QBrowseParams<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QContinuation<'a> { - context: YTContext<'a>, continuation: &'a str, } @@ -1126,18 +1129,17 @@ impl RustyPipeQuery { /// # Parameters /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) /// - `localized`: Whether to include the configured language and country - pub async fn get_context<'a>( + async fn get_context<'a>( &'a self, ctype: ClientType, localized: bool, - visitor_data: Option<&'a str>, + visitor_data: &'a str, ) -> YTContext { let (hl, gl) = if localized { (self.opts.lang, self.opts.country) } else { (Language::En, Country::Us) }; - let visitor_data = visitor_data.or(self.opts.visitor_data.as_deref()); match ctype { ClientType::Desktop => YTContext { @@ -1451,11 +1453,22 @@ impl RustyPipeQuery { ) -> Result { tracing::debug!("getting {}({})", operation, id); + let visitor_data = ctx_src + .visitor_data + .or(self.opts.visitor_data.as_deref()) + .map(Cow::Borrowed) + .unwrap_or_else(|| util::random_visitor_data(self.opts.country).into()); + + let context = self + .get_context(ctype, !ctx_src.unlocalized, &visitor_data) + .await; + let req_body = QBody { context, body }; + let ctx = MapRespCtx { id, lang: self.opts.lang, deobf: ctx_src.deobf, - visitor_data: ctx_src.visitor_data.or(self.opts.visitor_data.as_deref()), + visitor_data: Some(&visitor_data), client_type: ctype, artist: ctx_src.artist, }; @@ -1463,7 +1476,7 @@ impl RustyPipeQuery { let request = self .request_builder(ctype, endpoint, ctx.visitor_data) .await - .json(body) + .json(&req_body) .build()?; let req_res = self.yt_request::(&request, &ctx).await?; @@ -1563,47 +1576,6 @@ impl RustyPipeQuery { .await } - /* - /// Execute a request to the YouTube API, then map the response. - /// - /// Creates a report in case of failure for easy debugging. - /// - /// # Parameters - /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) - /// - `operation`: Name of the RustyPipe operation (only for reporting, e.g. `get_player`) - /// - `id`: ID of the requested entity (Video ID, Channel ID, ...). - /// The ID is included in reports and is also passed to the mapper for validating the response. - /// Set it to an empty string if you are not requesting an entity with an ID. - /// - `method`: HTTP method - /// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/?key=...`) - /// - `body`: Serializable request body to be sent in json format - /// - `visitor_data`: YouTube visitor data cookie - async fn execute_request_vdata< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( - &self, - ctype: ClientType, - operation: &str, - id: &str, - endpoint: &str, - body: &B, - visitor_data: Option<&str>, - ) -> Result { - self.execute_request_deobf::( - ctype, - operation, - id, - endpoint, - body, - visitor_data, - None, - ) - .await - } - */ - /// Execute a request to the YouTube API and return the response string /// /// # Parameters @@ -1616,10 +1588,20 @@ impl RustyPipeQuery { endpoint: &str, body: &B, ) -> Result { + let visitor_data = self + .opts + .visitor_data + .as_deref() + .map(Cow::Borrowed) + .unwrap_or_else(|| util::random_visitor_data(self.opts.country).into()); + + let context = self.get_context(ctype, true, &visitor_data).await; + let req_body = QBody { context, body }; + let request = self .request_builder(ctype, endpoint, None) .await - .json(body) + .json(&req_body) .build()?; self.client.http_request_txt(&request).await @@ -1646,6 +1628,7 @@ struct MapRespCtxSource<'a> { visitor_data: Option<&'a str>, deobf: Option<&'a DeobfData>, artist: Option, + unlocalized: bool, } impl<'a> MapRespCtx<'a> { @@ -1663,15 +1646,6 @@ impl<'a> MapRespCtx<'a> { } } -impl<'a> MapRespCtxSource<'a> { - fn visitor_data(visitor_data: &'a str) -> Self { - Self { - visitor_data: Some(visitor_data), - ..Default::default() - } - } -} - /// Implement this for YouTube API response structs that need to be mapped to /// RustyPipe models. trait MapResponse { diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 9c2bf94..5d72e84 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -41,9 +41,7 @@ impl RustyPipeQuery { } async fn _music_artist(&self, artist_id: &str, all_albums: bool) -> Result { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: artist_id, }; @@ -84,24 +82,18 @@ impl RustyPipeQuery { filter: Option, order: Option, ) -> Result, Error> { - let visitor_data = self.get_visitor_data().await?; - let context = self - .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) - .await; let request_body = QBrowseParams { - context: context.clone(), browse_id: &format!("{}{}", util::ARTIST_DISCOGRAPHY_PREFIX, artist_id), params: &albums_param(filter, order), }; let first_page = self - .execute_request_ctx::( + .execute_request::( ClientType::DesktopMusic, "music_artist_albums", artist_id, "browse", &request_body, - MapRespCtxSource::visitor_data(&visitor_data), ) .await?; @@ -109,10 +101,7 @@ impl RustyPipeQuery { let mut ctoken = first_page.ctoken; while let Some(tkn) = &ctoken { - let request_body = QContinuation { - context: context.clone(), - continuation: tkn, - }; + let request_body = QContinuation { continuation: tkn }; let resp: Paginator = self .execute_request_ctx::, _>( ClientType::DesktopMusic, @@ -122,7 +111,6 @@ impl RustyPipeQuery { &request_body, MapRespCtxSource { artist: Some(first_page.artist.clone()), - visitor_data: Some(&visitor_data), ..Default::default() }, ) diff --git a/src/client/music_charts.rs b/src/client/music_charts.rs index 7ad6149..2d21513 100644 --- a/src/client/music_charts.rs +++ b/src/client/music_charts.rs @@ -11,13 +11,12 @@ use crate::{ use super::{ response::{self, music_item::MusicListMapper, url_endpoint::MusicPageType}, - ClientType, MapRespCtx, MapResponse, RustyPipeQuery, YTContext, + ClientType, MapRespCtx, MapResponse, RustyPipeQuery, }; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QCharts<'a> { - context: YTContext<'a>, browse_id: &'a str, params: &'a str, #[serde(skip_serializing_if = "Option::is_none")] @@ -34,9 +33,7 @@ impl RustyPipeQuery { /// Get the YouTube Music charts for a given country #[tracing::instrument(skip(self), level = "error")] pub async fn music_charts(&self, country: Option) -> Result { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QCharts { - context, browse_id: "FEmusic_charts", params: "sgYPRkVtdXNpY19leHBsb3Jl", form_data: country.map(|c| FormData { diff --git a/src/client/music_details.rs b/src/client/music_details.rs index ca0630b..34351ff 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -16,12 +16,11 @@ use super::{ self, music_item::{map_queue_item, MusicListMapper}, }, - ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery, YTContext, + ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, }; #[derive(Debug, Serialize)] struct QMusicDetails<'a> { - context: YTContext<'a>, video_id: &'a str, enable_persistent_playlist_panel: bool, is_audio_only: bool, @@ -30,7 +29,6 @@ struct QMusicDetails<'a> { #[derive(Debug, Serialize)] struct QRadio<'a> { - context: YTContext<'a>, playlist_id: &'a str, params: &'a str, enable_persistent_playlist_panel: bool, @@ -46,9 +44,7 @@ impl RustyPipeQuery { video_id: S, ) -> Result { let video_id = video_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QMusicDetails { - context, video_id, enable_persistent_playlist_panel: true, is_audio_only: true, @@ -71,9 +67,7 @@ impl RustyPipeQuery { #[tracing::instrument(skip(self), level = "error")] pub async fn music_lyrics + Debug>(&self, lyrics_id: S) -> Result { let lyrics_id = lyrics_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: lyrics_id, }; @@ -96,9 +90,7 @@ impl RustyPipeQuery { related_id: S, ) -> Result { let related_id = related_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: related_id, }; @@ -121,12 +113,7 @@ impl RustyPipeQuery { radio_id: S, ) -> Result, Error> { let radio_id = radio_id.as_ref(); - let visitor_data = self.get_visitor_data().await?; - let context = self - .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) - .await; let request_body = QRadio { - context, playlist_id: radio_id, params: "wAEB8gECeAE%3D", enable_persistent_playlist_panel: true, @@ -134,13 +121,12 @@ impl RustyPipeQuery { tuner_setting_value: "AUTOMIX_SETTING_NORMAL", }; - self.execute_request_ctx::( + self.execute_request::( ClientType::DesktopMusic, "music_radio", radio_id, "next", &request_body, - MapRespCtxSource::visitor_data(&visitor_data), ) .await } diff --git a/src/client/music_genres.rs b/src/client/music_genres.rs index 627b93b..cfda986 100644 --- a/src/client/music_genres.rs +++ b/src/client/music_genres.rs @@ -15,9 +15,7 @@ impl RustyPipeQuery { /// Get a list of moods and genres from YouTube Music #[tracing::instrument(skip(self), level = "error")] pub async fn music_genres(&self) -> Result, Error> { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: "FEmusic_moods_and_genres", }; @@ -38,9 +36,7 @@ impl RustyPipeQuery { genre_id: S, ) -> Result { let genre_id = genre_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowseParams { - context, browse_id: "FEmusic_moods_and_genres_category", params: genre_id, }; diff --git a/src/client/music_new.rs b/src/client/music_new.rs index cfc5b6d..fb6cf17 100644 --- a/src/client/music_new.rs +++ b/src/client/music_new.rs @@ -13,9 +13,7 @@ impl RustyPipeQuery { /// Get the new albums that were released on YouTube Music #[tracing::instrument(skip(self), level = "error")] pub async fn music_new_albums(&self) -> Result, Error> { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: "FEmusic_new_releases_albums", }; @@ -32,9 +30,7 @@ impl RustyPipeQuery { /// Get the new music videos that were released on YouTube Music #[tracing::instrument(skip(self), level = "error")] pub async fn music_new_videos(&self) -> Result, Error> { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: "FEmusic_new_releases_videos", }; diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 3345651..0edde4f 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -17,7 +17,7 @@ use super::{ self, music_item::{map_album_type, map_artist_id, map_artists, MusicListMapper}, }, - ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery, + ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, }; impl RustyPipeQuery { @@ -28,30 +28,16 @@ impl RustyPipeQuery { playlist_id: S, ) -> Result { let playlist_id = playlist_id.as_ref(); - // YTM playlists require visitor data for continuations to work - let visitor_data = if playlist_id.starts_with("RD") { - Some(self.get_visitor_data().await?) - } else { - None - }; - let context = self - .get_context(ClientType::DesktopMusic, true, visitor_data.as_deref()) - .await; let request_body = QBrowse { - context, browse_id: &format!("VL{playlist_id}"), }; - self.execute_request_ctx::( + self.execute_request::( ClientType::DesktopMusic, "music_playlist", playlist_id, "browse", &request_body, - MapRespCtxSource { - visitor_data: visitor_data.as_deref(), - ..Default::default() - }, ) .await } @@ -63,9 +49,7 @@ impl RustyPipeQuery { album_id: S, ) -> Result { let album_id = album_id.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { - context, browse_id: album_id, }; diff --git a/src/client/music_search.rs b/src/client/music_search.rs index dfcc7f2..d8548ed 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -15,12 +15,11 @@ use crate::{ serializer::MapResult, }; -use super::{response, ClientType, MapRespCtx, MapResponse, RustyPipeQuery, YTContext}; +use super::{response, ClientType, MapRespCtx, MapResponse, RustyPipeQuery}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QSearch<'a> { - context: YTContext<'a>, query: &'a str, #[serde(skip_serializing_if = "Option::is_none")] params: Option<&'a str>, @@ -29,7 +28,6 @@ struct QSearch<'a> { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QSearchSuggestion<'a> { - context: YTContext<'a>, input: &'a str, } @@ -44,9 +42,7 @@ impl RustyPipeQuery { filter: Option, ) -> Result, Error> { let query = query.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QSearch { - context, query, params: filter.map(MusicSearchFilter::params), }; @@ -132,11 +128,7 @@ impl RustyPipeQuery { query: S, ) -> Result { let query = query.as_ref(); - let context = self.get_context(ClientType::DesktopMusic, true, None).await; - let request_body = QSearchSuggestion { - context, - input: query, - }; + let request_body = QSearchSuggestion { input: query }; self.execute_request::( ClientType::DesktopMusic, diff --git a/src/client/pagination.rs b/src/client/pagination.rs index b6e22d1..f656bab 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::fmt::Debug; use crate::error::{Error, ExtractionError}; @@ -25,16 +24,7 @@ impl RustyPipeQuery { ) -> Result, Error> { let ctoken = ctoken.as_ref(); if endpoint.is_music() { - // Visitor data is required for YTM continuations - let visitor_data = match visitor_data { - Some(vd) => Cow::Borrowed(vd), - None => Cow::Owned(self.get_visitor_data().await?), - }; - let context = self - .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) - .await; let request_body = QContinuation { - context, continuation: ctoken, }; @@ -45,59 +35,60 @@ impl RustyPipeQuery { ctoken, endpoint.as_str(), &request_body, - MapRespCtxSource::visitor_data(&visitor_data), + MapRespCtxSource { + visitor_data, + ..Default::default() + }, ) .await?; - Ok(map_ytm_paginator(p, Some(&visitor_data), endpoint)) + Ok(map_ytm_paginator(p, endpoint)) } else { - let context = self - .get_context(ClientType::Desktop, true, visitor_data) - .await; let request_body = QContinuation { - context, continuation: ctoken, }; let p = self - .execute_request::, _>( + .execute_request_ctx::, _>( ClientType::Desktop, "continuation", ctoken, endpoint.as_str(), &request_body, + MapRespCtxSource { + visitor_data, + ..Default::default() + }, ) .await?; - Ok(map_yt_paginator(p, visitor_data, endpoint)) + Ok(map_yt_paginator(p, endpoint)) } } } fn map_yt_paginator( p: Paginator, - visitor_data: Option<&str>, endpoint: ContinuationEndpoint, ) -> Paginator { Paginator { count: p.count, items: p.items.into_iter().filter_map(T::from_yt_item).collect(), ctoken: p.ctoken, - visitor_data: visitor_data.map(str::to_owned), + visitor_data: p.visitor_data, endpoint, } } fn map_ytm_paginator( p: Paginator, - visitor_data: Option<&str>, endpoint: ContinuationEndpoint, ) -> Paginator { Paginator { count: p.count, items: p.items.into_iter().filter_map(T::from_ytm_item).collect(), ctoken: p.ctoken, - visitor_data: visitor_data.map(str::to_owned), + visitor_data: p.visitor_data, endpoint, } } @@ -430,7 +421,7 @@ mod tests { let map_res: MapResult> = items.map_response(&MapRespCtx::test("")).unwrap(); let paginator: Paginator = - map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); + map_yt_paginator(map_res.c, ContinuationEndpoint::Browse); assert!( map_res.warnings.is_empty(), @@ -453,7 +444,7 @@ mod tests { let map_res: MapResult> = items.map_response(&MapRespCtx::test("")).unwrap(); let paginator: Paginator = - map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); + map_yt_paginator(map_res.c, ContinuationEndpoint::Browse); assert!( map_res.warnings.is_empty(), @@ -476,7 +467,7 @@ mod tests { let map_res: MapResult> = items.map_response(&MapRespCtx::test("")).unwrap(); let paginator: Paginator = - map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); + map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); assert!( map_res.warnings.is_empty(), @@ -497,7 +488,7 @@ mod tests { let map_res: MapResult> = items.map_response(&MapRespCtx::test("")).unwrap(); let paginator: Paginator = - map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); + map_ytm_paginator(map_res.c, ContinuationEndpoint::MusicBrowse); assert!( map_res.warnings.is_empty(), diff --git a/src/client/player.rs b/src/client/player.rs index 6a6e8a6..4da3162 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -24,14 +24,13 @@ use super::{ self, player::{self, Format}, }, - ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, RustyPipeQuery, YTContext, + ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, RustyPipeQuery, DEFAULT_PLAYER_CLIENT_ORDER, }; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QPlayer<'a> { - context: YTContext<'a>, /// Website playback context #[serde(skip_serializing_if = "Option::is_none")] playback_context: Option>, @@ -124,16 +123,10 @@ impl RustyPipeQuery { client_type: ClientType, ) -> Result { let video_id = video_id.as_ref(); - // let vdata = self.get_visitor_data().await?; - let (context, deobf) = tokio::join!( - self.get_context(client_type, false, None), - self.client.get_deobf_data() - ); - let deobf = deobf?; + let deobf = self.client.get_deobf_data().await?; let request_body = if client_type.is_web() { QPlayer { - context, playback_context: Some(QPlaybackContext { content_playback_context: QContentPlaybackContext { signature_timestamp: &deobf.sts, @@ -148,7 +141,6 @@ impl RustyPipeQuery { } } else { QPlayer { - context, playback_context: None, cpn: Some(util::generate_content_playback_nonce()), video_id, diff --git a/src/client/playlist.rs b/src/client/playlist.rs index f61b7ef..df000e8 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -13,40 +13,23 @@ use crate::{ util::{self, dictionary, timeago, TryRemove}, }; -use super::{ - response, ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, QBrowse, - RustyPipeQuery, -}; +use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { /// Get a YouTube playlist #[tracing::instrument(skip(self), level = "error")] pub async fn playlist + Debug>(&self, playlist_id: S) -> Result { let playlist_id = playlist_id.as_ref(); - // YTM playlists require visitor data for continuations to work - let visitor_data: Option = if playlist_id.starts_with("RD") { - Some(self.get_visitor_data().await?) - } else { - None - }; - let context = self - .get_context(ClientType::Desktop, true, visitor_data.as_deref()) - .await; let request_body = QBrowse { - context, browse_id: &format!("VL{playlist_id}"), }; - self.execute_request_ctx::( + self.execute_request::( ClientType::Desktop, "playlist", playlist_id, "browse", &request_body, - MapRespCtxSource { - visitor_data: visitor_data.as_deref(), - ..Default::default() - }, ) .await } diff --git a/src/client/search.rs b/src/client/search.rs index 50f3b33..042f200 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -12,12 +12,11 @@ use crate::{ param::search_filter::SearchFilter, }; -use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, RustyPipeQuery, YTContext}; +use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, RustyPipeQuery}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct QSearch<'a> { - context: YTContext<'a>, query: &'a str, params: &'a str, } @@ -30,9 +29,7 @@ impl RustyPipeQuery { query: S, ) -> Result, Error> { let query = query.as_ref(); - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QSearch { - context, query, params: "8AEB", }; @@ -55,9 +52,7 @@ impl RustyPipeQuery { filter: &SearchFilter, ) -> Result, Error> { let query = query.as_ref(); - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QSearch { - context, query, params: &filter.encode(), }; diff --git a/src/client/trends.rs b/src/client/trends.rs index c271b7d..fa3510c 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -12,9 +12,7 @@ impl RustyPipeQuery { /// Get the videos from the YouTube trending page #[tracing::instrument(skip(self), level = "error")] pub async fn trending(&self) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowseParams { - context, browse_id: "FEtrending", params: "4gIOGgxtb3N0X3BvcHVsYXI%3D", }; diff --git a/src/client/url_resolver.rs b/src/client/url_resolver.rs index e771d6e..40b1b66 100644 --- a/src/client/url_resolver.rs +++ b/src/client/url_resolver.rs @@ -11,13 +11,12 @@ use crate::{ use super::{ response::{self, url_endpoint::NavigationEndpoint}, - ClientType, MapRespCtx, MapResponse, RustyPipeQuery, YTContext, + ClientType, MapRespCtx, MapResponse, RustyPipeQuery, }; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct QResolveUrl<'a> { - context: YTContext<'a>, +struct QResolveUrl { url: String, } @@ -299,9 +298,7 @@ impl RustyPipeQuery { url_path: &str, ctype: ClientType, ) -> Result { - let context = self.get_context(ctype, true, None).await; let request_body = QResolveUrl { - context, url: format!( "https://{}.youtube.com{}", match ctype { diff --git a/src/client/video_details.rs b/src/client/video_details.rs index a1fb93e..f219531 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -15,12 +15,11 @@ use crate::{ use super::{ response::{self, video_details::Payload, IconType}, - ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery, YTContext, + ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery, }; #[derive(Debug, Serialize)] struct QVideo<'a> { - context: YTContext<'a>, /// YouTube video ID video_id: &'a str, /// Set to true to allow extraction of streams with sensitive content @@ -37,9 +36,7 @@ impl RustyPipeQuery { video_id: S, ) -> Result { let video_id = video_id.as_ref(); - let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QVideo { - context, video_id, content_check_ok: true, racy_check_ok: true, @@ -63,11 +60,7 @@ impl RustyPipeQuery { visitor_data: Option<&str>, ) -> Result, Error> { let ctoken = ctoken.as_ref(); - let context = self - .get_context(ClientType::Desktop, true, visitor_data) - .await; let request_body = QContinuation { - context, continuation: ctoken, }; diff --git a/src/util/mod.rs b/src/util/mod.rs index 121c1b0..0d36425 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -94,6 +94,29 @@ pub fn random_uuid() -> String { ) } +/// Generate a random visitor data cookie +pub fn random_visitor_data(country: Country) -> String { + let mut rng = rand::thread_rng(); + + let mut pb_e2 = ProtoBuilder::new(); + pb_e2.string(2, ""); + pb_e2.varint(4, rng.gen_range(1..256)); + + let mut pb_e = ProtoBuilder::new(); + pb_e.string(1, &country.to_string()); + pb_e.embedded(2, pb_e2); + + let mut pb = ProtoBuilder::new(); + pb.string(1, &random_string(CONTENT_PLAYBACK_NONCE_ALPHABET, 11)); + pb.varint( + 5, + (time::OffsetDateTime::now_utc().unix_timestamp() as u64) + .saturating_sub(rng.gen_range(0..600_000)), + ); + pb.embedded(6, pb_e); + pb.to_base64() +} + /// Split an URL into its base string and parameter map /// /// Example: