From 562ac2df7e24d8f98edab236f483e5644e3795ca Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 11 Oct 2022 18:49:15 +0200 Subject: [PATCH] fix: retry on empty continuation responses --- src/client/channel.rs | 22 +++--- src/client/mod.rs | 71 ++++++++++++++++--- src/client/playlist.rs | 7 +- src/client/response/channel.rs | 2 + src/client/response/playlist.rs | 1 + src/client/response/search.rs | 1 + src/client/response/video_details.rs | 12 +++- src/client/video_details.rs | 102 +++++++++++++-------------- src/error.rs | 2 + tests/youtube.rs | 10 +-- 10 files changed, 142 insertions(+), 88 deletions(-) diff --git a/src/client/channel.rs b/src/client/channel.rs index f9f9300..116504c 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -281,12 +281,11 @@ impl MapResponse> for response::ChannelCont { _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; - let res = some_or_bail!( - actions.try_swap_remove(0), - Err(ExtractionError::InvalidData("no received action".into())) - ) - .append_continuation_items_action - .continuation_items; + let res = actions + .try_swap_remove(0) + .ok_or(ExtractionError::Retry)? + .append_continuation_items_action + .continuation_items; Ok(map_videos(res, lang)) } @@ -300,12 +299,11 @@ impl MapResponse> for response::ChannelCont { _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; - let res = some_or_bail!( - actions.try_swap_remove(0), - Err(ExtractionError::InvalidData("no received action".into())) - ) - .append_continuation_items_action - .continuation_items; + let res = actions + .try_swap_remove(0) + .ok_or(ExtractionError::Retry)? + .append_continuation_items_action + .continuation_items; Ok(map_playlists(res)) } diff --git a/src/client/mod.rs b/src/client/mod.rs index 9d842ae..5b31877 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -169,7 +169,8 @@ struct RustyPipeRef { http: Client, storage: Option>, reporter: Option>, - n_retries: u32, + n_http_retries: u32, + n_query_retries: u32, consent_cookie: String, cache: CacheHolder, default_opts: RustyPipeOpts, @@ -186,7 +187,8 @@ struct RustyPipeOpts { pub struct RustyPipeBuilder { storage: Option>, reporter: Option>, - n_retries: u32, + n_http_retries: u32, + n_query_retries: u32, user_agent: String, default_opts: RustyPipeOpts, } @@ -277,7 +279,8 @@ impl RustyPipeBuilder { default_opts: RustyPipeOpts::default(), storage: Some(Box::new(FileStorage::default())), reporter: Some(Box::new(FileReporter::default())), - n_retries: 3, + n_http_retries: 3, + n_query_retries: 2, user_agent: DEFAULT_UA.to_owned(), } } @@ -312,7 +315,8 @@ impl RustyPipeBuilder { http, storage: self.storage, reporter: self.reporter, - n_retries: self.n_retries, + n_http_retries: self.n_http_retries, + n_query_retries: self.n_query_retries, consent_cookie: format!( "{}={}{}", CONSENT_COOKIE, @@ -367,8 +371,18 @@ impl RustyPipeBuilder { /// random jitter to be less predictable). /// /// **Default value**: 3 - pub fn n_retries(mut self, n_retries: u32) -> Self { - self.n_retries = n_retries; + pub fn n_http_retries(mut self, n_retries: u32) -> Self { + self.n_http_retries = n_retries; + self + } + + /// Set the number of retries for YouTube API queries. + /// + /// If a YouTube API requests returns invalid data, the request is repeated. + /// + /// **Default value**: 2 + pub fn n_query_retries(mut self, n_retries: u32) -> Self { + self.n_http_retries = n_retries; self } @@ -458,7 +472,7 @@ impl RustyPipe { request: Request, ) -> core::result::Result { let mut last_res = None; - for n in 0..self.inner.n_retries { + for n in 0..self.inner.n_http_retries { let res = self.inner.http.execute(request.try_clone().unwrap()).await; let emsg = match &res { Ok(response) => { @@ -939,6 +953,44 @@ impl RustyPipeQuery { endpoint: &str, body: &B, deobf: Option<&Deobfuscator>, + ) -> Result { + for n in 0..self.client.inner.n_query_retries.saturating_sub(1) { + let res = self + ._try_execute_request_deobf::(ctype, operation, id, endpoint, body, deobf) + .await; + let emsg = match res { + Ok(res) => return Ok(res), + Err(error) => match &error { + Error::Extraction(e) => match e { + ExtractionError::Deserialization(_) + | ExtractionError::InvalidData(_) + | ExtractionError::WrongResult(_) + | ExtractionError::Retry => e.to_string(), + _ => return Err(error), + }, + _ => return Err(error), + }, + }; + + warn!("{} retry attempt #{}. Error: {}.", operation, n, emsg); + } + self._try_execute_request_deobf::(ctype, operation, id, endpoint, body, deobf) + .await + } + + /// Single try of `execute_request_deobf` + async fn _try_execute_request_deobf< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + id: &str, + endpoint: &str, + body: &B, + deobf: Option<&Deobfuscator>, ) -> Result { let request = self .request_builder(ctype, endpoint) @@ -949,7 +1001,7 @@ impl RustyPipeQuery { let request_url = request.url().to_string(); let request_headers = request.headers().to_owned(); - let response = self.client.inner.http.execute(request).await?; + let response = self.client.http_request(request).await?; let status = response.status(); let resp_str = response.text().await?; @@ -1013,7 +1065,8 @@ impl RustyPipeQuery { ExtractionError::VideoUnavailable(_, _) | ExtractionError::VideoAgeRestricted | ExtractionError::ContentUnavailable(_) - | ExtractionError::NoData => (), + | ExtractionError::NoData + | ExtractionError::Retry => (), _ => create_report(Level::ERR, Some(e.to_string()), Vec::new()), } Err(e.into()) diff --git a/src/client/playlist.rs b/src/client/playlist.rs index f137a80..2db5079 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -193,12 +193,7 @@ impl MapResponse> for response::PlaylistCont { _deobf: Option<&Deobfuscator>, ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; - let action = some_or_bail!( - actions.try_swap_remove(0), - Err(ExtractionError::InvalidData( - "no continuation action".into() - )) - ); + let action = actions.try_swap_remove(0).ok_or(ExtractionError::Retry)?; let (items, ctoken) = map_playlist_items(action.append_continuation_items_action.continuation_items.c); diff --git a/src/client/response/channel.rs b/src/client/response/channel.rs index 9bfbe57..00c3e63 100644 --- a/src/client/response/channel.rs +++ b/src/client/response/channel.rs @@ -25,6 +25,8 @@ pub struct Channel { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ChannelCont { + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] pub on_response_received_actions: Vec, } diff --git a/src/client/response/playlist.rs b/src/client/response/playlist.rs index 7fe0d31..e26d6d4 100644 --- a/src/client/response/playlist.rs +++ b/src/client/response/playlist.rs @@ -19,6 +19,7 @@ pub struct Playlist { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PlaylistCont { + #[serde(default)] #[serde_as(as = "VecSkipError<_>")] pub on_response_received_actions: Vec, } diff --git a/src/client/response/search.rs b/src/client/response/search.rs index 9ebe29d..ce85a9b 100644 --- a/src/client/response/search.rs +++ b/src/client/response/search.rs @@ -25,6 +25,7 @@ pub struct Search { pub struct SearchCont { #[serde_as(as = "Option")] pub estimated_results: Option, + #[serde_as(as = "VecSkipError<_>")] pub on_response_received_commands: Vec, } diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index f453c56..fc3ca34 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -11,7 +11,8 @@ use crate::serializer::{ }; use super::{ - ContinuationEndpoint, ContinuationItemRenderer, Icon, Thumbnails, VideoListItem, VideoOwner, + ContinuationEndpoint, ContinuationItemRenderer, Icon, MusicContinuation, Thumbnails, + VideoListItem, VideoOwner, }; /* @@ -282,6 +283,8 @@ pub struct RecommendationResults { /// Can be `None` for age-restricted videos #[serde_as(as = "Option>")] pub results: Option>>, + #[serde_as(as = "Option>")] + pub continuations: Option>, } /// The engagement panels are displayed below the video and contain chapter markers @@ -418,9 +421,12 @@ pub struct CommentItemSectionHeaderMenuItem { */ /// Video recommendations continuation response +#[serde_as] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VideoRecommendations { + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] pub on_response_received_endpoints: Vec, } @@ -459,8 +465,8 @@ pub struct VideoComments { /// - Comment replies: appendContinuationItemsAction /// - n*commentRenderer, continuationItemRenderer: /// replies + continuation - #[serde_as(as = "VecLogError<_>")] - pub on_response_received_endpoints: MapResult>, + #[serde_as(as = "Option>")] + pub on_response_received_endpoints: Option>>, } /// Video comments continuation diff --git a/src/client/video_details.rs b/src/client/video_details.rs index e718c46..48c5b94 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -251,7 +251,7 @@ impl MapResponse for response::VideoDetails { .secondary_results .and_then(|sr| { sr.secondary_results.results.map(|r| { - let mut res = map_recommendations(r, lang); + let mut res = map_recommendations(r, sr.secondary_results.continuations, lang); warnings.append(&mut res.warnings); res.c }) @@ -342,15 +342,11 @@ impl MapResponse> for response::VideoRecommendations _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { let mut endpoints = self.on_response_received_endpoints; - let cont = some_or_bail!( - endpoints.try_swap_remove(0), - Err(ExtractionError::InvalidData( - "no continuation endpoint".into() - )) - ); + let cont = endpoints.try_swap_remove(0).ok_or(ExtractionError::Retry)?; Ok(map_recommendations( cont.append_continuation_items_action.continuation_items, + None, lang, )) } @@ -363,57 +359,54 @@ impl MapResponse> for response::VideoComments { lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { - let mut warnings = self.on_response_received_endpoints.warnings; + let received_endpoints = self + .on_response_received_endpoints + .ok_or(ExtractionError::Retry)?; + let mut warnings = received_endpoints.warnings; let mut comments = Vec::new(); let mut comment_count = None; let mut ctoken = None; - self.on_response_received_endpoints - .c - .into_iter() - .for_each(|citem| { - let mut items = citem.append_continuation_items_action.continuation_items; - warnings.append(&mut items.warnings); - items.c.into_iter().for_each(|item| match item { - response::video_details::CommentListItem::CommentThreadRenderer { - comment, - replies, + received_endpoints.c.into_iter().for_each(|citem| { + let mut items = citem.append_continuation_items_action.continuation_items; + warnings.append(&mut items.warnings); + items.c.into_iter().for_each(|item| match item { + response::video_details::CommentListItem::CommentThreadRenderer { + comment, + replies, + rendering_priority, + } => { + let mut res = map_comment( + comment.comment_renderer, + Some(replies), rendering_priority, - } => { - let mut res = map_comment( - comment.comment_renderer, - Some(replies), - rendering_priority, - lang, - ); - comments.push(res.c); - warnings.append(&mut res.warnings) - } - response::video_details::CommentListItem::CommentRenderer(comment) => { - let mut res = map_comment( - comment, - None, - response::video_details::CommentPriority::RenderingPriorityUnknown, - lang, - ); - comments.push(res.c); - warnings.append(&mut res.warnings) - } - response::video_details::CommentListItem::ContinuationItemRenderer { - continuation_endpoint, - } => { - ctoken = Some(continuation_endpoint.continuation_command.token); - } - response::video_details::CommentListItem::CommentsHeaderRenderer { - count_text, - } => { - comment_count = count_text.and_then(|txt| { - util::parse_numeric_or_warn::(&txt, &mut warnings) - }); - } - }); + lang, + ); + comments.push(res.c); + warnings.append(&mut res.warnings) + } + response::video_details::CommentListItem::CommentRenderer(comment) => { + let mut res = map_comment( + comment, + None, + response::video_details::CommentPriority::RenderingPriorityUnknown, + lang, + ); + comments.push(res.c); + warnings.append(&mut res.warnings) + } + response::video_details::CommentListItem::ContinuationItemRenderer { + continuation_endpoint, + } => { + ctoken = Some(continuation_endpoint.continuation_command.token); + } + response::video_details::CommentListItem::CommentsHeaderRenderer { count_text } => { + comment_count = count_text + .and_then(|txt| util::parse_numeric_or_warn::(&txt, &mut warnings)); + } }); + }); Ok(MapResult { c: Paginator::new(comment_count, comments, ctoken), @@ -424,6 +417,7 @@ impl MapResponse> for response::VideoComments { fn map_recommendations( r: MapResult>, + continuations: Option>, lang: Language, ) -> MapResult> { let mut warnings = r.warnings; @@ -475,6 +469,12 @@ fn map_recommendations( }) .collect::>(); + if let Some(continuations) = continuations { + continuations.into_iter().for_each(|c| { + ctoken = Some(c.next_continuation_data.continuation); + }) + }; + MapResult { c: Paginator::new(None, items, ctoken), warnings, diff --git a/src/error.rs b/src/error.rs index e690178..8d34a10 100644 --- a/src/error.rs +++ b/src/error.rs @@ -87,6 +87,8 @@ pub enum ExtractionError { WrongResult(String), #[error("Warnings during deserialization/mapping")] DeserializationWarnings, + #[error("Got no data from YouTube, attempt retry")] + Retry, } /// Internal error diff --git a/tests/youtube.rs b/tests/youtube.rs index 373399a..ed9cab4 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -19,7 +19,7 @@ use rustypipe::param::{ #[case::tv_html5_embed(ClientType::TvHtml5Embed)] #[case::android(ClientType::Android)] #[case::ios(ClientType::Ios)] -#[test_log::test(tokio::test)] +#[tokio::test] async fn get_player(#[case] client_type: ClientType) { let rp = RustyPipe::builder().strict().build(); let player_data = rp.query().player("n4tK7LYFxI0", client_type).await.unwrap(); @@ -179,7 +179,7 @@ async fn get_playlist( assert!(!playlist.thumbnail.is_empty()); } -#[test_log::test(tokio::test)] +#[tokio::test] async fn playlist_cont() { let rp = RustyPipe::builder().strict().build(); let mut playlist = rp @@ -197,7 +197,7 @@ async fn playlist_cont() { assert!(playlist.videos.count.unwrap() > 100); } -#[test_log::test(tokio::test)] +#[tokio::test] async fn playlist_cont2() { let rp = RustyPipe::builder().strict().build(); let mut playlist = rp @@ -311,7 +311,6 @@ async fn get_video_details_music() { assert!(!details.is_live); assert!(!details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // Comments are disabled for this video @@ -369,7 +368,6 @@ async fn get_video_details_ccommons() { assert!(!details.is_live); assert!(details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -506,7 +504,6 @@ async fn get_video_details_chapters() { ] "###); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); assert!( @@ -566,7 +563,6 @@ async fn get_video_details_live() { assert!(details.is_live); assert!(!details.is_ccommons); - assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); // No comments because livestream