From 3a0db09e239287579aaefc9c98808504dda6dea3 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 17 Oct 2022 23:32:12 +0200 Subject: [PATCH] fix: add visitor_data to recommendation paginator --- src/client/mod.rs | 14 +- src/client/pagination.rs | 136 ++++++++++-------- src/client/response/mod.rs | 6 + src/client/response/trends.rs | 8 +- src/client/response/video_details.rs | 3 +- ...s__map_video_details_20220924_newdesc.snap | 1 + ...deo_details_20221011_new_continuation.snap | 1 + ...s__map_video_details_20221011_rec_isr.snap | 1 + ...ls__tests__map_video_details_ccommons.snap | 1 + ...ls__tests__map_video_details_chapters.snap | 1 + ...etails__tests__map_video_details_live.snap | 1 + ...tails__tests__map_video_details_music.snap | 1 + ..._details__tests__map_video_details_mv.snap | 1 + src/client/trends.rs | 8 +- src/client/video_details.rs | 32 +++-- src/model/mod.rs | 1 - src/model/paginator.rs | 43 +----- src/param/mod.rs | 20 +++ 18 files changed, 155 insertions(+), 124 deletions(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index 1ada971..b4e64b3 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -291,8 +291,8 @@ impl RustyPipeBuilder { default_opts: RustyPipeOpts::default(), storage: Some(Box::new(FileStorage::default())), reporter: Some(Box::new(FileReporter::default())), - n_http_retries: 3, - n_query_retries: 2, + n_http_retries: 2, + n_query_retries: 1, user_agent: DEFAULT_UA.to_owned(), } } @@ -382,7 +382,7 @@ impl RustyPipeBuilder { /// The waiting time is doubled for subsequent attempts (including a bit of /// random jitter to be less predictable). /// - /// **Default value**: 3 + /// **Default value**: 2 pub fn n_http_retries(mut self, n_retries: u32) -> Self { self.n_http_retries = n_retries; self @@ -392,9 +392,9 @@ impl RustyPipeBuilder { /// /// If a YouTube API requests returns invalid data, the request is repeated. /// - /// **Default value**: 2 + /// **Default value**: 1 pub fn n_query_retries(mut self, n_retries: u32) -> Self { - self.n_http_retries = n_retries; + self.n_query_retries = n_retries; self } @@ -481,7 +481,7 @@ impl RustyPipe { /// Execute the given http request. async fn http_request(&self, request: Request) -> Result { let mut last_res = None; - for n in 0..self.inner.n_http_retries { + for n in 0..=self.inner.n_http_retries { let res = self.inner.http.execute(request.try_clone().unwrap()).await; let emsg = match &res { Ok(response) => { @@ -975,7 +975,7 @@ impl RustyPipeQuery { body: &B, deobf: Option<&Deobfuscator>, ) -> Result { - for n in 0..self.client.inner.n_query_retries.saturating_sub(1) { + for n in 0..self.client.inner.n_query_retries { let res = self ._try_execute_request_deobf::( ctype, diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 5917e6a..ea19bb0 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,5 +1,6 @@ use crate::error::{Error, ExtractionError}; -use crate::model::{Comment, ContinuationEndpoint, Paginator, PlaylistVideo, YouTubeItem}; +use crate::model::{Comment, Paginator, PlaylistVideo, YouTubeItem}; +use crate::param::ContinuationEndpoint; use crate::serializer::MapResult; use crate::util::TryRemove; @@ -80,65 +81,6 @@ impl> MapResponse> for response::Continuati } } -macro_rules! paginator { - ($entity_type:ty, $cont_function:path) => { - impl Paginator<$entity_type> { - pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { - Ok(match &self.ctoken { - Some(ctoken) => Some($cont_function(query, ctoken).await?), - None => None, - }) - } - - pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { - match self.next(query).await { - Ok(Some(paginator)) => { - let mut items = paginator.items; - self.items.append(&mut items); - self.ctoken = paginator.ctoken; - Ok(true) - } - Ok(None) => Ok(false), - Err(e) => Err(e), - } - } - - pub async fn extend_pages( - &mut self, - query: RustyPipeQuery, - n_pages: usize, - ) -> Result<(), Error> { - for _ in 0..n_pages { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - - pub async fn extend_limit( - &mut self, - query: RustyPipeQuery, - n_items: usize, - ) -> Result<(), Error> { - while self.items.len() < n_items { - match self.extend(query.clone()).await { - Ok(false) => break, - Err(e) => return Err(e), - _ => {} - } - } - Ok(()) - } - } - }; -} - -paginator!(Comment, RustyPipeQuery::video_comments); -paginator!(PlaylistVideo, RustyPipeQuery::playlist_continuation); - impl> Paginator { pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { Ok(match &self.ctoken { @@ -195,6 +137,80 @@ impl> Paginator { } } +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + Ok(match &self.ctoken { + Some(ctoken) => Some( + query + .video_comments(ctoken, self.visitor_data.as_deref()) + .await?, + ), + _ => None, + }) + } +} + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result, Error> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.playlist_continuation(ctoken).await?), + None => None, + }) + } +} + +macro_rules! paginator { + ($entity_type:ty) => { + impl Paginator<$entity_type> { + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages( + &mut self, + query: RustyPipeQuery, + n_pages: usize, + ) -> Result<(), Error> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit( + &mut self, + query: RustyPipeQuery, + n_items: usize, + ) -> Result<(), Error> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + } + }; +} + +paginator!(Comment); +paginator!(PlaylistVideo); + #[cfg(test)] mod tests { use std::{fs::File, io::BufReader, path::Path}; diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index b94cff3..aaeea7e 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -193,6 +193,12 @@ pub struct ContinuationAction { pub continuation_items: MapResult>, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResponseContext { + pub visitor_data: Option, +} + // YouTube Music #[serde_as] diff --git a/src/client/response/trends.rs b/src/client/response/trends.rs index a807083..94e7960 100644 --- a/src/client/response/trends.rs +++ b/src/client/response/trends.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use serde_with::{serde_as, VecSkipError}; -use super::{video_item::YouTubeListRendererWrap, ContentRenderer}; +use super::{video_item::YouTubeListRendererWrap, ContentRenderer, ResponseContext}; #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -35,9 +35,3 @@ pub struct BrowseResults { pub struct Tab { pub tab_renderer: ContentRenderer, } - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ResponseContext { - pub visitor_data: Option, -} diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index c487116..06a268b 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -10,11 +10,11 @@ use crate::serializer::{ MapResult, VecLogError, }; -use super::YouTubeListItem; use super::{ url_endpoint::BrowseEndpoint, ContinuationEndpoint, ContinuationItemRenderer, Icon, MusicContinuation, Thumbnails, VideoOwner, }; +use super::{ResponseContext, YouTubeListItem}; /* #VIDEO DETAILS @@ -32,6 +32,7 @@ pub struct VideoDetails { /// Video chapters + comment section #[serde_as(as = "VecLogError<_>")] pub engagement_panels: MapResult>, + pub response_context: ResponseContext, } /// Video details main object, contains video metadata and recommended videos diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap index 3974513..9f857b2 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20220924_newdesc.snap @@ -735,6 +735,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILWmVlcnJudUxpNUXAAQHIAQEYACreBjJzNkw2d3pfQkFyOEJBb0Q4ajRBQ2czQ1Bnb0l5dFdIekt5Tm1ZMXBDZ1B5UGdBS0o5SS1KQW9pVUV4eU1UUldiR05sYkRKVE16UlpSMmw1WlZkRU1rOW9jbU4wVGt4Q1psVk9PUW9EOGo0QUNoTFNQZzhLRFZKRVdtVmxjbkp1ZFV4cE5VVUtBX0ktQUFvT3dqNExDTjIyMHJ2Q3h2cVNsQUVLQV9JLUFBb3cwajR0Q2l0U1JFTk1RVXMxZFhsZmF6STNkWFV0UlhSUlgySTFWVEp5TWpaRVRrUmFUMjFPY1Vka1kyTlZTVWRSQ2dQeVBnQUtEc0ktQ3dqZnZzMkxuNFRwbmJRQkNnUHlQZ0FLRGNJLUNnallxdldKeExEazhYc0tBX0ktQUFvT3dqNExDTlNUMDZfUXlOdjZxUUVLQV9JLUFBb093ajRMQ1AyMThJbnd0cldWdHdFS0FfSS1BQW9Od2o0S0NJbmp5ZmpWdHVMMVh3b0Q4ajRBQ2czQ1Bnb0l4LUM1djltdnBwZ3lDZ1B5UGdBS0RzSS1Dd2kwMnZQMC10ZTBydGdCQ2dQeVBnQUtEY0ktQ2dqUDNLU2s3Nno4OVFrS0FfSS1BQW9Od2o0S0NMZTVyN3p2MGVTRWJ3b0Q4ajRBQ2czQ1Bnb0kyNXVaaTVYU2dPY0lDZ1B5UGdBS0RzSS1Dd2lEb1k3dXR2RFp3WW9CQ2dQeVBnQUtEY0ktQ2dqMXRLMkVxY1RtM3dRS0FfSS1BQW9Od2o0S0NNdnRtWl9hZ29TUEpnb0Q4ajRBQ2czQ1Bnb0l1UFdDZ09mWDFmdFlDZ1B5UGdBS0RjSS1DZ2k0dHNfbnpMZWozbWNTRkFBQ0JBWUlDZ3dPRUJJVUZoZ2FIQjRnSWlRbUdnUUlBQkFCR2dRSUFoQURHZ1FJQkJBRkdnUUlCaEFIR2dRSUNCQUpHZ1FJQ2hBTEdnUUlEQkFOR2dRSURoQVBHZ1FJRUJBUkdnUUlFaEFUR2dRSUZCQVZHZ1FJRmhBWEdnUUlHQkFaR2dRSUdoQWJHZ1FJSEJBZEdnUUlIaEFmR2dRSUlCQWhHZ1FJSWhBakdnUUlKQkFsR2dRSUpoQW5LaFFBQWdRR0NBb01EaEFTRkJZWUdod2VJQ0lrSmdqD3dhdGNoLW5leHQtZmVlZA%3D%3D"), + visitor_data: Some("CgtCeURHR09uNlJ5TSjOiLqZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap index 85eb65a..bf01573 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_new_continuation.snap @@ -809,6 +809,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILWmVlcnJudUxpNUXAAQHIAQEYACqiDDJzNkw2d3lTQ1FxUENRb0Q4ajRBQ2czQ1Bnb0l1UFdDZ09mWDFmdFlDZ1B5UGdBS0RzSS1Dd2pPcjZhVTlMN2ttdUVCQ2dQeVBnQUtFdEktRHdvTlVrUmFaV1Z5Y201MVRHazFSUW9EOGo0QUNnN0NQZ3NJLU1xaTZ1MlZ3NC01QVFvRDhqNEFDZzNDUGdvSXNZamU0NGJFeGFKUkNnUHlQZ0FLRGNJLUNnaXF4bzYxd01DQ3d6WUtBX0ktQUFvT3dqNExDTmVSckxfYzNNaTEzd0VLQV9JLUFBb053ajRLQ051Ym1ZdVYwb0RuQ0FvRDhqNEFDZzNDUGdvSTE2YTg4NTI1OXNsUkNnUHlQZ0FLRGNJLUNnamJqcXVGb2V1YjB3Z0tBX0ktQUFvTndqNEtDTW4xbEkzbHUtaW1mQW9EOGo0QUNnM0NQZ29JdXFpZTZ0SzRrZUZqQ2dQeVBnQUtEY0ktQ2dqUGdaejB2OC1sNkhRS0FfSS1BQW9Pd2o0TENQMjE4SW53dHJXVnR3RUtBX0ktQUFvT3dqNExDTXVLdF9DUDllR21nUUVLQV9JLUFBb053ajRLQ0szRXN0V3YwTC1iVWdvRDhqNEFDZzNDUGdvSWc1NzdoSnJSdDRvcUNnUHlQZ0FLRGNJLUNnaVJ1c1BtemZtenlGd0tBX0ktQUFvT3dqNExDTlRBcHMtZGh2eXEwZ0VLQV9JLUFBb053ajRLQ0p2aDhzV0g1OXk1SUFvRDhqNEFDaF9TUGh3S0dsSkVRVTk2ZFZaM1JWbDNZMUZFTkhWMmNHZEJUbU5JU0ZWM0NoX1NQaHdLR2xKRVFVOWZjQzFWYmpCSGVUbHVXRlZ0Wm1kaE0xTlNYMXAzQ2hfU1Bod0tHbEpFUVU5cFEwaENhR3R1VTBSd09HcFZWekJPUzFoWU9FMVJDaF9TUGh3S0dsSkVRVTlYWjBsd1lVbDZha1p6UVhkeE5GOXhWR2hyTlROQkNoX1NQaHdLR2xKRVFVOXFNVkpuZEhkZmVtZHJibWxSWkdkTU5XTnlVRmxCQ2hfU1Bod0tHbEpFUVU4NWExbHRhMU5KVG5CVGFYVldTalEzUjNkT1RWSm5DaF9TUGh3S0dsSkVRVTlGYjNOTlVtbHlhM1ZKTjNvNE1tSmZia0oyUjNoQkNoX1NQaHdLR2xKRVFVOW1PRlExTURaUVZGcFVWRmxDWm01RVRVNURiR0ZSQ2hfU1Bod0tHbEpFUVU5UllqSlhRWGxLYTBwMlRURmhaMGRYZEhkRkxVOUJDaF9TUGh3S0dsSkVRVTlNWDFGNk1scFJRbUZNUkROTFExTnFWalpYZG5wM0NoX1NQaHdLR2xKRVFVOXpjeTFGWVdSRFpHZzBUVmxYV0hsMGFtWkpabFYzQ2hfU1Bod0tHbEpFUVU4MVRraFVXblJGV0ROSGJIWlhRMjgyYTJOdGFrdDNDaF9TUGh3S0dsSkVRVTlMWDBjMVRVZzFaM0ZJUTNRd1VXdENZVlZJTjJwUkNoX1NQaHdLR2xKRVFVOVpUWEZhWlV4U1RXMXhaRW8zZGs5b09UQXRhME5CQ2hfU1Bod0tHbEpFUVU4eVNGbEJhMFpIYzBGSmFWVmthRE5NVUhGRE5UZG5FaFVBQWdRR0NBb01EaEFTRkJZWUdod2VJQ0lrSmlnYUJBZ0FFQUVhQkFnQ0VBTWFCQWdFRUFVYUJBZ0dFQWNhQkFnSUVBa2FCQWdLRUFzYUJBZ01FQTBhQkFnT0VBOGFCQWdRRUJFYUJBZ1NFQk1hQkFnVUVCVWFCQWdXRUJjYUJBZ1lFQmthQkFnYUVCc2FCQWdjRUIwYUJBZ2VFQjhhQkFnZ0VDRWFCQWdpRUNNYUJBZ2tFQ1VhQkFnbUVDY2FCQWdvRUNrYUJBZ29FQ29hQkFnb0VDc2FCQWdvRUN3YUJBZ29FQzBhQkFnb0VDNGFCQWdvRUM4YUJBZ29FREFhQkFnb0VERWFCQWdvRURJYUJBZ29FRE1hQkFnb0VEUWFCQWdvRURVYUJBZ29FRFlhQkFnb0VEY3FGUUFDQkFZSUNnd09FQklVRmhnYUhCNGdJaVFtS0FqD3dhdGNoLW5leHQtZmVlZA%3D%3D"), + visitor_data: Some("Cgs2V0p6ZW5ab1ozTSjkrpaaBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap index 764e6cb..7b53949 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_20221011_rec_isr.snap @@ -1267,6 +1267,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILbkZEQnhCVWZFNzTAAQHIAQEYACqIBjJzNkw2d3lfQkFxOEJBb0Q4ajRBQ2c3Q1Bnc0ltdG1tX1p6ei1xYTNBUW9EOGo0QUNnN0NQZ3NJcThTNW5lQ1N0c2JpQVFvRDhqNEFDZzNDUGdvSXlvel8tN21NbWI1TUNnUHlQZ0FLRGNJLUNnaTFyOUdxODh6aXoxQUtBX0ktQUFvT3dqNExDS0c4MVlXQWlLRFd5UUVLQV9JLUFBb093ajRMQ0xMUnRJMzRfYjNDeXdFS0FfSS1BQW9Od2o0S0NQZlcxdldzM3AyLWJ3b0Q4ajRBQ2czQ1Bnb0loOVhCZ2NtcjNvY3RDZ1B5UGdBS0RjSS1DZ2oxMHFycnU2VzdyRmtLQV9JLUFBb093ajRMQ05McHBmckhxdkh0dmdFS0FfSS1BQW9Od2o0S0NPbUdpTG13M09iUUp3b0Q4ajRBQ2czQ1Bnb0lySjc1czZ6TGd1VUtDZ1B5UGdBS0RzSS1Dd2lLdmZ1WTdJcmZuX1VCQ2dQeVBnQUtEc0ktQ3dqTzY5WHk1UDZhdnVRQkNnUHlQZ0FLRHNJLUN3aS1pOTN2OFkzM3NOY0JDZ1B5UGdBS0RjSS1DZ2pIbmNQR2dwakp2QkFLQV9JLUFBb013ajRKQ0tlSTRxUzl1dHB6Q2dQeVBnQUtEY0ktQ2dqVXJkbU83YWFLbVFrS0FfSS1BQW9Pd2o0TENPQ0xrT0hDdllxSjNRRUtBX0ktQUFvTndqNEtDUFRwazh6RXc2Yk1IUklVQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZYUJBZ0FFQUVhQkFnQ0VBTWFCQWdFRUFVYUJBZ0dFQWNhQkFnSUVBa2FCQWdLRUFzYUJBZ01FQTBhQkFnT0VBOGFCQWdRRUJFYUJBZ1NFQk1hQkFnVUVCVWFCQWdXRUJjYUJBZ1lFQmthQkFnYUVCc2FCQWdjRUIwYUJBZ2VFQjhhQkFnZ0VDRWFCQWdpRUNNYUJBZ2tFQ1VhQkFnbUVDY3FGQUFDQkFZSUNnd09FQklVRmhnYUhCNGdJaVFtag93YXRjaC1uZXh0LWZlZWQ%3D"), + visitor_data: Some("Cgtidzg4MlRTb3FKSSiqipeaBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap index e8109e3..c58a8d4 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_ccommons.snap @@ -756,6 +756,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILMHJiOUNmT3ZvamvAAQHIAQEYACqrBjJzNkw2d3paQkFyV0JBb0Q4ajRBQ2c3Q1Bnc0l4Ty1xb3AyV25NWDVBUW9EOGo0QUNnN0NQZ3NJZ29uTDc3clgtWjdqQVFvRDhqNEFDZzNDUGdvSTU5N2RnZW1vaEl4YUNnUHlQZ0FLSWRJLUhnb2NVa1JEVFZWRE1sUlljVjkwTURaSWFtUnlNbWRmUzJSTGNFaFJad29EOGo0QUNnN0NQZ3NJMDlqVG05MzIwZEhtQVFvRDhqNEFDZzdDUGdzSWphem5fUFhDNnF2c0FRb0Q4ajRBQ2c3Q1Bnc0lrckN0cmRULXdfdldBUW9EOGo0QUNnN0NQZ3NJdGZfQnJ0NjNuYjJPQVFvRDhqNEFDZzdDUGdzSW1wYnF5SkdsNmRudkFRb0Q4ajRBQ2c3Q1Bnc0lxT2FZbXBqZjM3ZTdBUW9EOGo0QUNnN0NQZ3NJMEtfcWhNRGYzZDI2QVFvRDhqNEFDZzNDUGdvSW45N1lqLWllbTdnLUNnUHlQZ0FLRHNJLUN3aVg5by1DNzhxbzBNa0JDZ1B5UGdBS0RzSS1Dd2o2a3BYUXNPS1otZFFCQ2dQeVBnQUtEc0ktQ3dpUTI2aVNxYjYyd0lnQkNnUHlQZ0FLRGNJLUNnanV5ZmUtLW9iRWlqNEtBX0ktQUFvTndqNEtDSWVGemRXOGpyMmRid29EOGo0QUNnM0NQZ29JMGRlT2tfNmlfZkloQ2dQeVBnQUtEc0ktQ3dpajRPenpwdnp5NUlJQkNnUHlQZ0FLRHNJLUN3akRqZkdfdXZYQm9MZ0JFaFFBQWdRR0NBb01EaEFTRkJZWUdod2VJQ0lrSmhvRUNBQVFBUm9FQ0FJUUF4b0VDQVFRQlJvRUNBWVFCeG9FQ0FnUUNSb0VDQW9RQ3hvRUNBd1FEUm9FQ0E0UUR4b0VDQkFRRVJvRUNCSVFFeG9FQ0JRUUZSb0VDQllRRnhvRUNCZ1FHUm9FQ0JvUUd4b0VDQndRSFJvRUNCNFFIeG9FQ0NBUUlSb0VDQ0lRSXhvRUNDUVFKUm9FQ0NZUUp5b1VBQUlFQmdnS0RBNFFFaFFXR0JvY0hpQWlKQ1lqD3dhdGNoLW5leHQtZmVlZA%3D%3D"), + visitor_data: Some("CgtoY1pQUF8wNW1qayjSjpSZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap index 82f59cc..5444508 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_chapters.snap @@ -1224,6 +1224,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILbkZEQnhCVWZFNzTAAQHIAQEYACqkBjJzNkw2d3pVQkFyUkJBb0Q4ajRBQ2d6Q1Bna0lwNGppcEwyNjJuTUtBX0ktQUFvTndqNEtDSW1CN18yaHlQUEdDUW9EOGo0QUNnM0NQZ29JaDlYQmdjbXIzb2N0Q2dQeVBnQUtEc0ktQ3dpZHRjTHlpWV9FaVpvQkNnUHlQZ0FLRHNJLUN3aW4wN2ZZbWZIVjVkVUJDZ1B5UGdBS0RjSS1DZ2pqNEstdl9ibWY0Z2dLQV9JLUFBb053ajRLQ0tqeDJfakowT0tlZlFvRDhqNEFDZzdDUGdzSXZvdmQ3X0dOOTdEWEFRb0Q4ajRBQ2czQ1Bnb0k0Y1hvektyVzFMWkJDZ1B5UGdBS0RjSS1DZ2pndWNfcXk4YkVneVFLQV9JLUFBb053ajRLQ1B2ZjAtcXMxdE90WlFvRDhqNEFDaUhTUGg0S0hGSkVRMDFWUTFoMWNWTkNiRWhCUlRaWWR5MTVaVXBCTUZSMWJuY0tBX0ktQUFvT3dqNExDTDZsdFl2ejZaQ2gyZ0VLQV9JLUFBb093ajRMQ0l6cnFkenM3NmJZMGdFS0FfSS1BQW9Pd2o0TENJUFNuOGpBbmRYYnNRRUtBX0ktQUFvT3dqNExDSXZEcXZidW9jamp6UUVLQV9JLUFBb093ajRMQ05HSXFzZVM5cUR2cFFFS0FfSS1BQW9Pd2o0TENNT2o4T3FwMVpIWDJBRUtBX0ktQUFvT3dqNExDSnJacHYyYzhfcW10d0VLQV9JLUFBb053ajRLQ1BmRGpKaTZzXy1ZUVJJVUFBSUVCZ2dLREE0UUVoUVdHQm9jSGlBaUpDWWFCQWdBRUFFYUJBZ0NFQU1hQkFnRUVBVWFCQWdHRUFjYUJBZ0lFQWthQkFnS0VBc2FCQWdNRUEwYUJBZ09FQThhQkFnUUVCRWFCQWdTRUJNYUJBZ1VFQlVhQkFnV0VCY2FCQWdZRUJrYUJBZ2FFQnNhQkFnY0VCMGFCQWdlRUI4YUJBZ2dFQ0VhQkFnaUVDTWFCQWdrRUNVYUJBZ21FQ2NxRkFBQ0JBWUlDZ3dPRUJJVUZoZ2FIQjRnSWlRbWoPd2F0Y2gtbmV4dC1mZWVk"), + visitor_data: Some("CgtIV0JjSUtDQm9LQSjUjpSZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap index c14349c..62803cc 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_live.snap @@ -815,6 +815,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILODZZTEZPb2c0R03AAQHIAQEYACrgBzJzNkw2d3poQlFyZUJRb0Q4ajRBQ2czQ1Bnb0k3b3VlbjdUTV9yRklDZ1B5UGdBS0RjSS1DZ2lIaU55ejlkLWI2M1VLQV9JLUFBb093ajRMQ0tXNTlNR2pwdkNhb0FFS0FfSS1BQW9Od2o0S0NLLTduYXJ2NXN1bWFRb0Q4ajRBQ2c3Q1Bnc0l2ZV9nLVBfQ3dPUHFBUW9EOGo0QUNnN0NQZ3NJdDRDUW1aS2hpTUdmQVFvRDhqNEFDZzNDUGdvSWpZQ1lnWVg4c1lOdUNnUHlQZ0FLRHNJLUN3aUFvckdHN3RtSW43Z0JDZ1B5UGdBS0RjSS1DZ2k0NmNINDBLZTYwR2NLQV9JLUFBb093ajRMQ0xLaHc5M1d1OUdKMWdFS0FfSS1BQW9Od2o0S0NJQ18ySnEzbHFDbVpBb0Q4ajRBQ2czQ1Bnb0l1SWlsckpyb2dxODBDZ1B5UGdBS0RzSS1Dd2lQN1lqano5X25pYW9CQ2dQeVBnQUtEc0ktQ3dqWG9ZVzc2ZUgyX3MwQkNnUHlQZ0FLRHNJLUN3ajU4ZmZxLVpHYnpwZ0JDZ1B5UGdBS0RjSS1DZ2pfNXEyR3k2djQzMzRLQV9JLUFBb053ajRLQ09hQWlMdncyZXFQSmdvRDhqNEFDZzNDUGdvSWhPMjJ3TXU2cWY0UUNnUHlQZ0FLRHNJLUN3ajluTnVJd016eDB1d0JDZ1B5UGdBS0RjSS1DZ2lLNU5PTjY5N0dtaWtLQV9JLUFBb2FtajhYQ2hWUGZEazRNek0xTXpjMU5ERTJOVGMzT0RnMk1UY0tHNW9fR0FvV1Qzd3hNekF5TXpnek9UY3hPREk0TXpVMU9EUTFNd29ibWo4WUNoWlBmREUxT0RVek5Ua3dPRGczTURVeE1qVTBOemcwQ2dQeVBnQUtBX0ktQUJJWEFBSUVCZ2dLREE0UUVoUVdHQm9jSGlBaUpDWW9MQzBhQkFnQUVBRWFCQWdDRUFNYUJBZ0VFQVVhQkFnR0VBY2FCQWdJRUFrYUJBZ0tFQXNhQkFnTUVBMGFCQWdPRUE4YUJBZ1FFQkVhQkFnU0VCTWFCQWdVRUJVYUJBZ1dFQmNhQkFnWUVCa2FCQWdhRUJzYUJBZ2NFQjBhQkFnZUVCOGFCQWdnRUNFYUJBZ2lFQ01hQkFna0VDVWFCQWdtRUNjYUJBZ29FQ2thQkFnb0VDb2FCQWdvRUNzYUJBZ3NFQ2thQkFnc0VDb2FCQWdzRUNzYUJBZ3RFQ2thQkFndEVDb2FCQWd0RUNzcUZ3QUNCQVlJQ2d3T0VCSVVGaGdhSEI0Z0lpUW1LQ3d0ag93YXRjaC1uZXh0LWZlZWQ%3D"), + visitor_data: Some("CgtnQS1WdzlNNkNCSSiSmKiZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap index 57a5ff7..b6ee3a8 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_music.snap @@ -561,6 +561,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILWHVNMm9uTUd2VEnAAQHIAQEYACqsBzJzNkw2d3k2QlFxM0JRb0Q4ajRBQ2czQ1Bnb0lvSmVsMDhiajMtcGVDZ1B5UGdBS0o5SS1KQW9pVUV4aGFGSktXRWczV0ZCSWJWZE9TemMwVm5wM1NVWXljWFZYY1dSR1NWZDJjZ29EOGo0QUNpZlNQaVFLSWxCTU5pMTRZV3hIWVZaRmRYQjJlVGxHUVVoZlJYZ3pTRlYxWHpGaFVrRXlialFLQV9JLUFBb1MwajRQQ2cxU1JGaDFUVEp2YmsxSGRsUkpDZ1B5UGdBS0RjSS1DZ2kxajRmWmtKNmo1UVVLQV9JLUFBb053ajRLQ0l6bHViS2doTldnSVFvRDhqNEFDZzNDUGdvSWpaeW4tUHlrXy1sU0NnUHlQZ0FLRHNJLUN3andyOUMtc18tb2g3SUJDZ1B5UGdBS0o5SS1KQW9pVUV3d1dqSlNSVFZsY1dGWlRWRXhiRnBaZGtWalRrOW1UVmx3VFVvelkyTktRd29EOGo0QUNnN0NQZ3NJM1lxcjFyWEI4TEs3QVFvRDhqNEFDZzdDUGdzSWhaYl83dVdna1BDZ0FRb0Q4ajRBQ2c3Q1Bnc0l2YjdxdUtldHhNYWtBUW9EOGo0QUNnM0NQZ29JblpEVXc5akYtT1VPQ2dQeVBnQUtIOUktSEFvYVVrUkZUV1E0VUZwSmRqbERVSE4yZGtWRVltOWZjRlZFTkhjS0FfSS1BQW9Od2o0S0NMLV9fclRrM3BmdENnb0Q4ajRBQ2c3Q1Bnc0lpZG5aNkxmZG5iZkxBUW9EOGo0QUNnM0NQZ29JdlB5dmw4UzJ4b0ppQ2dQeVBnQUtETUktQ1FpdHdOTzB0TmFrQ1FvRDhqNEFDaWZTUGlRS0lsQk1NazR6TjFoVWQxaG5jRlZpYWxSemNXVjFNbEZFZVRCTGIyRnpkMDlZTVZvS0FfSS1BQW9Od2o0S0NNQ1VudGpKOVkyaE94SVVBQUlFQmdnS0RBNFFFaFFXR0JvY0hpQWlKQ1lhQkFnQUVBRWFCQWdDRUFNYUJBZ0VFQVVhQkFnR0VBY2FCQWdJRUFrYUJBZ0tFQXNhQkFnTUVBMGFCQWdPRUE4YUJBZ1FFQkVhQkFnU0VCTWFCQWdVRUJVYUJBZ1dFQmNhQkFnWUVCa2FCQWdhRUJzYUJBZ2NFQjBhQkFnZUVCOGFCQWdnRUNFYUJBZ2lFQ01hQkFna0VDVWFCQWdtRUNjcUZBQUNCQVlJQ2d3T0VCSVVGaGdhSEI0Z0lpUW1qD3dhdGNoLW5leHQtZmVlZA%3D%3D"), + visitor_data: Some("CgtzclhqZVpoajVhVSi76qeZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap index 05ca067..7bd9112 100644 --- a/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap +++ b/src/client/snapshots/rustypipe__client__video_details__tests__map_video_details_mv.snap @@ -772,6 +772,7 @@ VideoDetails( ), ], ctoken: Some("CBQSExILWmVlcnJudUxpNUXAAQHIAQEYACq7BjJzNkw2d3psQkFyaUJBb0Q4ajRBQ2c3Q1Bnc0l6cS1tbFBTLTVKcmhBUW9EOGo0QUNoTFNQZzhLRFZKRVdtVmxjbkp1ZFV4cE5VVUtBX0ktQUFvdzBqNHRDaXRTUkVOTVFVczFkWGxmYXpJM2RYVXRSWFJSWDJJMVZUSnlNalpFVGtSYVQyMU9jVWRrWTJOVlNVZFJDZ1B5UGdBS0RjSS1DZ2k0OVlLQTU5ZlYtMWdLQV9JLUFBb053ajRLQ0tyR2pyWEF3SUxETmdvRDhqNEFDZzNDUGdvSWc1NzdoSnJSdDRvcUNnUHlQZ0FLRGNJLUNnakw3Wm1mMm9LRWp5WUtBX0ktQUFvTndqNEtDTnVibVl1VjBvRG5DQW9EOGo0QUNnN0NQZ3NJLU1xaTZ1MlZ3NC01QVFvRDhqNEFDZzNDUGdvSXk4ZmVyNE9zMHVSQ0NnUHlQZ0FLRGNJLUNnalByYkhnb292TTFSRUtBX0ktQUFvT3dqNExDSlhQa191MzVmVHJwQUVLQV9JLUFBb053ajRLQ0o3WmxkTG1pWkxDZFFvRDhqNEFDZzdDUGdzSWc2R083cmJ3MmNHS0FRb0Q4ajRBQ2czQ1Bnb0lyY1N5MWFfUXY1dFNDZ1B5UGdBS0RjSS1DZ2kzdWEtODc5SGtoRzhLQV9JLUFBb093ajRMQ0pfTTJhZUktNWZ4NmdFS0FfSS1BQW9Od2o0S0NMcW9udXJTdUpIaFl3b0Q4ajRBQ2c3Q1Bnc0l0TnJ6OVByWHRLN1lBUW9EOGo0QUNnM0NQZ29JOVpHYjR0LTZyYWMxRWhRQUFnUUdDQW9NRGhBU0ZCWVlHaHdlSUNJa0pob0VDQUFRQVJvRUNBSVFBeG9FQ0FRUUJSb0VDQVlRQnhvRUNBZ1FDUm9FQ0FvUUN4b0VDQXdRRFJvRUNBNFFEeG9FQ0JBUUVSb0VDQklRRXhvRUNCUVFGUm9FQ0JZUUZ4b0VDQmdRR1JvRUNCb1FHeG9FQ0J3UUhSb0VDQjRRSHhvRUNDQVFJUm9FQ0NJUUl4b0VDQ1FRSlJvRUNDWVFKeW9VQUFJRUJnZ0tEQTRRRWhRV0dCb2NIaUFpSkNZag93YXRjaC1uZXh0LWZlZWQ%3D"), + visitor_data: Some("Cgtjemd0bDVxU1N1QSjRjpSZBg%3D%3D"), endpoint: next, ), top_comments: Paginator( diff --git a/src/client/trends.rs b/src/client/trends.rs index 9926181..d98b23b 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -103,7 +103,13 @@ fn map_startpage_videos( mapper.map_response(videos); MapResult { - c: Paginator::new_with_vdata(None, mapper.items, mapper.ctoken, visitor_data), + c: Paginator::new_ext( + None, + mapper.items, + mapper.ctoken, + visitor_data, + crate::param::ContinuationEndpoint::Browse, + ), warnings: mapper.warnings, } } diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 0dc2d6e..0bfa01e 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -45,8 +45,13 @@ impl RustyPipeQuery { .await } - pub async fn video_comments(self, ctoken: &str) -> Result, Error> { - let context = self.get_context(ClientType::Desktop, true).await; + pub async fn video_comments( + self, + ctoken: &str, + visitor_data: Option<&str>, + ) -> Result, Error> { + let mut context = self.get_context(ClientType::Desktop, true).await; + context.client.visitor_data = visitor_data.map(str::to_owned); let request_body = QContinuation { context, continuation: ctoken, @@ -234,7 +239,12 @@ impl MapResponse for response::VideoDetails { .secondary_results .and_then(|sr| { sr.secondary_results.results.map(|r| { - let mut res = map_recommendations(r, sr.secondary_results.continuations, lang); + let mut res = map_recommendations( + r, + sr.secondary_results.continuations, + self.response_context.visitor_data, + lang, + ); warnings.append(&mut res.warnings); res.c }) @@ -309,17 +319,19 @@ impl MapResponse for response::VideoDetails { is_ccommons, chapters, recommended, - top_comments: Paginator::new_with_endpoint( + top_comments: Paginator::new_ext( comment_count, Vec::new(), comment_ctoken, - crate::model::ContinuationEndpoint::Next, + None, + crate::param::ContinuationEndpoint::Next, ), - latest_comments: Paginator::new_with_endpoint( + latest_comments: Paginator::new_ext( comment_count, Vec::new(), latest_comments_ctoken, - crate::model::ContinuationEndpoint::Next, + None, + crate::param::ContinuationEndpoint::Next, ), }, warnings, @@ -393,6 +405,7 @@ impl MapResponse> for response::VideoComments { fn map_recommendations( r: MapResult>, continuations: Option>, + visitor_data: Option, lang: Language, ) -> MapResult> { let mut mapper = response::YouTubeListMapper::::new(lang); @@ -405,11 +418,12 @@ fn map_recommendations( }; MapResult { - c: Paginator::new_with_endpoint( + c: Paginator::new_ext( None, mapper.items, mapper.ctoken, - crate::model::ContinuationEndpoint::Next, + visitor_data, + crate::param::ContinuationEndpoint::Next, ), warnings: mapper.warnings, } diff --git a/src/model/mod.rs b/src/model/mod.rs index 04b42d9..4807b85 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -4,7 +4,6 @@ mod ordering; mod paginator; pub mod richtext; -pub use paginator::ContinuationEndpoint; pub use paginator::Paginator; use std::ops::Range; diff --git a/src/model/paginator.rs b/src/model/paginator.rs index 40c6a5f..366541a 100644 --- a/src/model/paginator.rs +++ b/src/model/paginator.rs @@ -2,6 +2,8 @@ use std::convert::TryInto; use serde::{Deserialize, Serialize}; +use crate::param::ContinuationEndpoint; + /// Wrapper around progressively fetched items /// /// The paginator is a wrapper around a list of items that are fetched @@ -35,24 +37,6 @@ pub struct Paginator { pub endpoint: ContinuationEndpoint, } -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum ContinuationEndpoint { - Browse, - Search, - Next, -} - -impl ContinuationEndpoint { - pub(crate) fn as_str(self) -> &'static str { - match self { - ContinuationEndpoint::Browse => "browse", - ContinuationEndpoint::Search => "search", - ContinuationEndpoint::Next => "next", - } - } -} - impl Default for Paginator { fn default() -> Self { Self { @@ -67,31 +51,14 @@ impl Default for Paginator { impl Paginator { pub(crate) fn new(count: Option, items: Vec, ctoken: Option) -> Self { - Self::new_with_vdata(count, items, ctoken, None) + Self::new_ext(count, items, ctoken, None, ContinuationEndpoint::Browse) } - pub(crate) fn new_with_vdata( + pub(crate) fn new_ext( count: Option, items: Vec, ctoken: Option, visitor_data: Option, - ) -> Self { - Self { - count: match ctoken { - Some(_) => count, - None => items.len().try_into().ok(), - }, - items, - ctoken, - visitor_data, - endpoint: ContinuationEndpoint::Browse, - } - } - - pub(crate) fn new_with_endpoint( - count: Option, - items: Vec, - ctoken: Option, endpoint: ContinuationEndpoint, ) -> Self { Self { @@ -101,7 +68,7 @@ impl Paginator { }, items, ctoken, - visitor_data: None, + visitor_data, endpoint, } } diff --git a/src/param/mod.rs b/src/param/mod.rs index a339cfa..e5b69f3 100644 --- a/src/param/mod.rs +++ b/src/param/mod.rs @@ -4,6 +4,7 @@ pub mod locale; pub mod search_filter; pub use locale::{Country, Language}; +use serde::{Deserialize, Serialize}; pub use stream_filter::StreamFilter; /// Channel video sort order @@ -18,3 +19,22 @@ pub enum ChannelOrder { /// Output the most viewed videos first Popular, } + +/// YouTube API endpoint to fetch continuations from +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ContinuationEndpoint { + Browse, + Search, + Next, +} + +impl ContinuationEndpoint { + pub(crate) fn as_str(self) -> &'static str { + match self { + ContinuationEndpoint::Browse => "browse", + ContinuationEndpoint::Search => "search", + ContinuationEndpoint::Next => "next", + } + } +}