From 9d0ae0e9c24a3ddc47d3ef7214d600a51ba3333a Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 24 Oct 2022 07:45:57 +0200 Subject: [PATCH] feat: add visitor data parameter --- cli/src/main.rs | 2 +- codegen/src/download_testfiles.rs | 16 +++++----- src/client/channel.rs | 12 ++++---- src/client/channel_rss.rs | 2 +- src/client/mod.rs | 49 +++++++++++++++++++++++++------ src/client/pagination.rs | 33 +++++++++++---------- src/client/player.rs | 6 ++-- src/client/playlist.rs | 8 ++--- src/client/search.rs | 10 +++---- src/client/trends.rs | 8 ++--- src/client/url_resolver.rs | 2 +- src/client/video_details.rs | 11 +++---- tests/youtube.rs | 25 +++++++++++----- 13 files changed, 113 insertions(+), 71 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index ec603f8..6ab09a9 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -145,7 +145,7 @@ async fn download_playlist( let mut playlist = rp.query().playlist(id).await.unwrap(); playlist .videos - .extend_pages(rp.query(), usize::MAX) + .extend_pages(&rp.query(), usize::MAX) .await .unwrap(); diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 58a19d3..6a83f99 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -158,7 +158,7 @@ async fn playlist_cont(testfiles: &Path) { .unwrap(); let rp = rp_testfile(&json_path); - playlist.videos.next(rp.query()).await.unwrap().unwrap(); + playlist.videos.next(&rp.query()).await.unwrap().unwrap(); } async fn video_details(testfiles: &Path) { @@ -196,7 +196,7 @@ async fn comments_top(testfiles: &Path) { let rp = rp_testfile(&json_path); details .top_comments - .next(rp.query()) + .next(&rp.query()) .await .unwrap() .unwrap(); @@ -216,7 +216,7 @@ async fn comments_latest(testfiles: &Path) { let rp = rp_testfile(&json_path); details .latest_comments - .next(rp.query()) + .next(&rp.query()) .await .unwrap() .unwrap(); @@ -234,7 +234,7 @@ async fn recommendations(testfiles: &Path) { let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); let rp = rp_testfile(&json_path); - details.recommended.next(rp.query()).await.unwrap(); + details.recommended.next(&rp.query()).await.unwrap(); } async fn channel_videos(testfiles: &Path) { @@ -304,7 +304,7 @@ async fn channel_videos_cont(testfiles: &Path) { .unwrap(); let rp = rp_testfile(&json_path); - videos.content.next(rp.query()).await.unwrap().unwrap(); + videos.content.next(&rp.query()).await.unwrap().unwrap(); } async fn channel_playlists_cont(testfiles: &Path) { @@ -323,7 +323,7 @@ async fn channel_playlists_cont(testfiles: &Path) { .unwrap(); let rp = rp_testfile(&json_path); - playlists.content.next(rp.query()).await.unwrap().unwrap(); + playlists.content.next(&rp.query()).await.unwrap().unwrap(); } async fn search(testfiles: &Path) { @@ -350,7 +350,7 @@ async fn search_cont(testfiles: &Path) { let search = rp.query().search("doobydoobap").await.unwrap(); let rp = rp_testfile(&json_path); - search.items.next(rp.query()).await.unwrap().unwrap(); + search.items.next(&rp.query()).await.unwrap().unwrap(); } async fn search_playlists(testfiles: &Path) { @@ -412,7 +412,7 @@ async fn startpage_cont(testfiles: &Path) { let startpage = rp.query().startpage().await.unwrap(); let rp = rp_testfile(&json_path); - startpage.next(rp.query()).await.unwrap(); + startpage.next(&rp.query()).await.unwrap(); } async fn trending(testfiles: &Path) { diff --git a/src/client/channel.rs b/src/client/channel.rs index 6b90a7f..17ee868 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -39,7 +39,7 @@ enum Params { impl RustyPipeQuery { pub async fn channel_videos( - self, + &self, channel_id: &str, ) -> Result>, Error> { self.channel_videos_ordered(channel_id, ChannelOrder::default()) @@ -47,11 +47,11 @@ impl RustyPipeQuery { } pub async fn channel_videos_ordered( - self, + &self, channel_id: &str, order: ChannelOrder, ) -> Result>, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { context, browse_id: channel_id, @@ -73,10 +73,10 @@ impl RustyPipeQuery { } pub async fn channel_playlists( - self, + &self, channel_id: &str, ) -> Result>, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { context, browse_id: channel_id, @@ -94,7 +94,7 @@ impl RustyPipeQuery { } pub async fn channel_info(&self, channel_id: &str) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QChannel { context, browse_id: channel_id, diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index e7d3307..0cb9322 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -9,7 +9,7 @@ use crate::{ use super::{response, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn channel_rss(self, channel_id: &str) -> Result { + pub async fn channel_rss(&self, channel_id: &str) -> Result { let url = format!( "https://www.youtube.com/feeds/videos.xml?channel_id={}", channel_id diff --git a/src/client/mod.rs b/src/client/mod.rs index e762253..d5c82d5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -45,7 +45,7 @@ use crate::{ /// - **Desktop**: used by youtube.com /// - **DesktopMusic**: used by music.youtube.com, can access special music data, /// cannot access non-music content -/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos +/// - **TvHtml5Embed**: used by Smart TVs, can access age-restricted videos /// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams /// - **Ios**: used by the iOS app, no obfuscated URLs #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] @@ -69,7 +69,7 @@ impl ClientType { #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct YTContext<'a> { +pub struct YTContext<'a> { client: ClientInfo<'a>, /// only used on desktop #[serde(skip_serializing_if = "Option::is_none")] @@ -93,7 +93,7 @@ struct ClientInfo<'a> { #[serde(skip_serializing_if = "Option::is_none")] original_url: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] - visitor_data: Option, + visitor_data: Option<&'a str>, hl: Language, gl: Country, } @@ -183,6 +183,7 @@ struct RustyPipeRef { reporter: Option>, n_http_retries: u32, consent_cookie: String, + visitor_data: Option, cache: CacheHolder, default_opts: RustyPipeOpts, } @@ -200,6 +201,7 @@ pub struct RustyPipeBuilder { reporter: Option>, n_http_retries: u32, user_agent: String, + visitor_data: Option, default_opts: RustyPipeOpts, } @@ -292,6 +294,7 @@ impl RustyPipeBuilder { reporter: Some(Box::new(FileReporter::default())), n_http_retries: 2, user_agent: DEFAULT_UA.to_owned(), + visitor_data: None, } } @@ -332,6 +335,7 @@ impl RustyPipeBuilder { CONSENT_COOKIE_YES, rand::thread_rng().gen_range(100..1000) ), + visitor_data: self.visitor_data, cache: CacheHolder { desktop_client: RwLock::new(cdata.desktop_client), music_client: RwLock::new(cdata.music_client), @@ -434,6 +438,11 @@ impl RustyPipeBuilder { self.default_opts.strict = true; self } + + pub fn visitor_data(mut self, visitor_data: &str) -> Self { + self.visitor_data = Some(visitor_data.to_owned()); + self + } } impl Default for RustyPipe { @@ -745,7 +754,12 @@ impl RustyPipeQuery { /// # Parameters /// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...) /// - `localized`: Whether to include the configured language and country - async fn get_context(&self, ctype: ClientType, localized: bool) -> YTContext { + pub async fn get_context<'a>( + &'a self, + ctype: ClientType, + localized: bool, + visitor_data: Option<&'a str>, + ) -> YTContext { let hl = match localized { true => self.opts.lang, false => Language::En, @@ -754,6 +768,7 @@ impl RustyPipeQuery { true => self.opts.country, false => Country::Us, }; + let visitor_data = self.client.inner.visitor_data.as_deref().or(visitor_data); match ctype { ClientType::Desktop => YTContext { @@ -764,7 +779,7 @@ impl RustyPipeQuery { device_model: None, platform: "DESKTOP", original_url: Some("https://www.youtube.com/"), - visitor_data: None, + visitor_data, hl, gl, }, @@ -780,7 +795,7 @@ impl RustyPipeQuery { device_model: None, platform: "DESKTOP", original_url: Some("https://music.youtube.com/"), - visitor_data: None, + visitor_data, hl, gl, }, @@ -796,7 +811,7 @@ impl RustyPipeQuery { device_model: None, platform: "TV", original_url: None, - visitor_data: None, + visitor_data, hl, gl, }, @@ -814,7 +829,7 @@ impl RustyPipeQuery { device_model: None, platform: "MOBILE", original_url: None, - visitor_data: None, + visitor_data, hl, gl, }, @@ -830,7 +845,7 @@ impl RustyPipeQuery { device_model: Some(IOS_DEVICE_MODEL), platform: "MOBILE", original_url: None, - visitor_data: None, + visitor_data, hl, gl, }, @@ -1085,6 +1100,22 @@ impl RustyPipeQuery { self.execute_request_deobf::(ctype, operation, id, endpoint, body, None) .await } + + /// Execute a request to the YouTube API and return the response string + pub async fn raw( + &self, + ctype: ClientType, + endpoint: &str, + body: &B, + ) -> Result { + let request = self + .request_builder(ctype, endpoint) + .await + .json(body) + .build()?; + + self.client.http_request_txt(request).await + } } /// Implement this for YouTube API response structs that need to be mapped to diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 818d3af..579b181 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -10,13 +10,14 @@ use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery}; impl RustyPipeQuery { pub async fn continuation>( - self, + &self, ctoken: &str, endpoint: ContinuationEndpoint, visitor_data: Option<&str>, ) -> Result, Error> { - let mut context = self.get_context(ClientType::Desktop, true).await; - context.client.visitor_data = visitor_data.map(str::to_owned); + let context = self + .get_context(ClientType::Desktop, true, visitor_data) + .await; let request_body = QContinuation { context, continuation: ctoken, @@ -87,7 +88,7 @@ impl> MapResponse> for response::Continuati } impl> Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + pub async fn next(&self, query: &RustyPipeQuery) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some( query @@ -98,7 +99,7 @@ impl> Paginator { }) } - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + pub async fn extend(&mut self, query: &RustyPipeQuery) -> Result { match self.next(query).await { Ok(Some(paginator)) => { let mut items = paginator.items; @@ -113,11 +114,11 @@ impl> Paginator { pub async fn extend_pages( &mut self, - query: RustyPipeQuery, + query: &RustyPipeQuery, n_pages: usize, ) -> Result<(), Error> { for _ in 0..n_pages { - match self.extend(query.clone()).await { + match self.extend(query).await { Ok(false) => break, Err(e) => return Err(e), _ => {} @@ -128,11 +129,11 @@ impl> Paginator { pub async fn extend_limit( &mut self, - query: RustyPipeQuery, + query: &RustyPipeQuery, n_items: usize, ) -> Result<(), Error> { while self.items.len() < n_items { - match self.extend(query.clone()).await { + match self.extend(query).await { Ok(false) => break, Err(e) => return Err(e), _ => {} @@ -143,7 +144,7 @@ impl> Paginator { } impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + pub async fn next(&self, query: &RustyPipeQuery) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some( query @@ -156,7 +157,7 @@ impl Paginator { } impl Paginator { - pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + pub async fn next(&self, query: &RustyPipeQuery) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), None => None, @@ -167,7 +168,7 @@ impl Paginator { macro_rules! paginator { ($entity_type:ty) => { impl Paginator<$entity_type> { - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + pub async fn extend(&mut self, query: &RustyPipeQuery) -> Result { match self.next(query).await { Ok(Some(paginator)) => { let mut items = paginator.items; @@ -182,11 +183,11 @@ macro_rules! paginator { pub async fn extend_pages( &mut self, - query: RustyPipeQuery, + query: &RustyPipeQuery, n_pages: usize, ) -> Result<(), Error> { for _ in 0..n_pages { - match self.extend(query.clone()).await { + match self.extend(query).await { Ok(false) => break, Err(e) => return Err(e), _ => {} @@ -197,11 +198,11 @@ macro_rules! paginator { pub async fn extend_limit( &mut self, - query: RustyPipeQuery, + query: &RustyPipeQuery, n_items: usize, ) -> Result<(), Error> { while self.items.len() < n_items { - match self.extend(query.clone()).await { + match self.extend(query).await { Ok(false) => break, Err(e) => return Err(e), _ => {} diff --git a/src/client/player.rs b/src/client/player.rs index bdfdd56..66417c5 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -58,7 +58,7 @@ struct QContentPlaybackContext { } impl RustyPipeQuery { - pub async fn player(self, video_id: &str) -> Result { + pub async fn player(&self, video_id: &str) -> Result { let q1 = self.clone(); let android_res = q1.player_from_client(video_id, ClientType::Android).await; @@ -75,12 +75,12 @@ impl RustyPipeQuery { } pub async fn player_from_client( - self, + &self, video_id: &str, client_type: ClientType, ) -> Result { let (context, deobf) = tokio::join!( - self.get_context(client_type, false), + self.get_context(client_type, false, None), self.client.get_deobf() ); let deobf = deobf?; diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 73180f6..9161847 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -14,8 +14,8 @@ use crate::{ use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn playlist(self, playlist_id: &str) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn playlist(&self, playlist_id: &str) -> Result { + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, browse_id: "VL".to_owned() + playlist_id, @@ -32,10 +32,10 @@ impl RustyPipeQuery { } pub async fn playlist_continuation( - self, + &self, ctoken: &str, ) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QContinuation { context, continuation: ctoken, diff --git a/src/client/search.rs b/src/client/search.rs index e206589..2243302 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -21,8 +21,8 @@ struct QSearch<'a> { } impl RustyPipeQuery { - pub async fn search(self, query: &str) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn search(&self, query: &str) -> Result { + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QSearch { context, query, @@ -40,11 +40,11 @@ impl RustyPipeQuery { } pub async fn search_filter( - self, + &self, query: &str, filter: &SearchFilter, ) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QSearch { context, query, @@ -61,7 +61,7 @@ impl RustyPipeQuery { .await } - pub async fn search_suggestion(self, query: &str) -> Result, Error> { + pub async fn search_suggestion(&self, query: &str) -> Result, Error> { let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t", &[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.to_string())] ).map_err(|_| Error::Other("could not build url".into()))?; diff --git a/src/client/trends.rs b/src/client/trends.rs index da0bd60..26a59a8 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -11,8 +11,8 @@ use crate::{ use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn startpage(self) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn startpage(&self) -> Result, Error> { + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, browse_id: "FEwhat_to_watch".to_owned(), @@ -28,8 +28,8 @@ impl RustyPipeQuery { .await } - pub async fn trending(self) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn trending(&self) -> Result, Error> { + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { context, browse_id: "FEtrending".to_owned(), diff --git a/src/client/url_resolver.rs b/src/client/url_resolver.rs index d5ee925..c4f5a8e 100644 --- a/src/client/url_resolver.rs +++ b/src/client/url_resolver.rs @@ -165,7 +165,7 @@ impl RustyPipeQuery { } async fn _navigation_resolve_url(&self, url_path: &str) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QResolveUrl { context, url: format!("https://www.youtube.com{}", url_path), diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 714875e..dc62436 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -28,8 +28,8 @@ struct QVideo<'a> { } impl RustyPipeQuery { - pub async fn video_details(self, video_id: &str) -> Result { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn video_details(&self, video_id: &str) -> Result { + let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QVideo { context, video_id, @@ -48,12 +48,13 @@ impl RustyPipeQuery { } pub async fn video_comments( - self, + &self, ctoken: &str, visitor_data: Option<&str>, ) -> Result, Error> { - let mut context = self.get_context(ClientType::Desktop, true).await; - context.client.visitor_data = visitor_data.map(str::to_owned); + let context = self + .get_context(ClientType::Desktop, true, visitor_data) + .await; let request_body = QContinuation { context, continuation: ctoken, diff --git a/tests/youtube.rs b/tests/youtube.rs index b2c31ba..f2f0860 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -338,7 +338,7 @@ async fn playlist_cont() { playlist .videos - .extend_pages(rp.query(), usize::MAX) + .extend_pages(&rp.query(), usize::MAX) .await .unwrap(); assert!(playlist.videos.items.len() > 100); @@ -354,7 +354,11 @@ async fn playlist_cont2() { .await .unwrap(); - playlist.videos.extend_limit(rp.query(), 101).await.unwrap(); + playlist + .videos + .extend_limit(&rp.query(), 101) + .await + .unwrap(); assert!(playlist.videos.items.len() > 100); assert!(playlist.videos.count.unwrap() > 100); } @@ -797,7 +801,12 @@ async fn get_video_details_not_found() { async fn get_video_recommendations() { let rp = RustyPipe::builder().strict().build(); let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); - let next_recommendations = details.recommended.next(rp.query()).await.unwrap().unwrap(); + let next_recommendations = details + .recommended + .next(&rp.query()) + .await + .unwrap() + .unwrap(); // dbg!(&next_recommendations); assert!( @@ -815,7 +824,7 @@ async fn get_video_comments() { let top_comments = details .top_comments - .next(rp.query()) + .next(&rp.query()) .await .unwrap() .unwrap(); @@ -837,7 +846,7 @@ async fn get_video_comments() { let latest_comments = details .latest_comments - .next(rp.query()) + .next(&rp.query()) .await .unwrap() .unwrap(); @@ -892,7 +901,7 @@ async fn channel_videos(#[case] order: ChannelOrder) { _ => unimplemented!(), } - let next = channel.content.next(rp.query()).await.unwrap().unwrap(); + let next = channel.content.next(&rp.query()).await.unwrap().unwrap(); assert!( !next.is_exhausted() && !next.items.is_empty(), "no more videos" @@ -915,7 +924,7 @@ async fn channel_playlists() { "got no playlists" ); - let next = channel.content.next(rp.query()).await.unwrap().unwrap(); + let next = channel.content.next(&rp.query()).await.unwrap().unwrap(); assert!( !next.is_exhausted() && !next.items.is_empty(), "no more playlists" @@ -1267,7 +1276,7 @@ async fn startpage_cont() { let rp = RustyPipe::builder().strict().build(); let startpage = rp.query().startpage().await.unwrap(); - let next = startpage.next(rp.query()).await.unwrap().unwrap(); + let next = startpage.next(&rp.query()).await.unwrap().unwrap(); assert!( next.items.len() >= 20,