From ae91c46fb21e603009e7785ddfe231e35287bf5c Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 17 Oct 2022 23:42:10 +0200 Subject: [PATCH] refactor: remove query retries --- src/client/mod.rs | 121 ++++-------------- src/client/pagination.rs | 20 +-- src/client/playlist.rs | 4 +- src/client/response/mod.rs | 4 +- src/client/response/video_details.rs | 4 +- src/client/video_details.rs | 4 +- src/error.rs | 2 - .../video_details/recommendations_empty.json | 94 -------------- 8 files changed, 33 insertions(+), 220 deletions(-) delete mode 100644 testfiles/video_details/recommendations_empty.json diff --git a/src/client/mod.rs b/src/client/mod.rs index b4e64b3..3001afa 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -182,7 +182,6 @@ struct RustyPipeRef { storage: Option>, reporter: Option>, n_http_retries: u32, - n_query_retries: u32, consent_cookie: String, cache: CacheHolder, default_opts: RustyPipeOpts, @@ -200,7 +199,6 @@ pub struct RustyPipeBuilder { storage: Option>, reporter: Option>, n_http_retries: u32, - n_query_retries: u32, user_agent: String, default_opts: RustyPipeOpts, } @@ -292,7 +290,6 @@ impl RustyPipeBuilder { storage: Some(Box::new(FileStorage::default())), reporter: Some(Box::new(FileReporter::default())), n_http_retries: 2, - n_query_retries: 1, user_agent: DEFAULT_UA.to_owned(), } } @@ -328,7 +325,6 @@ impl RustyPipeBuilder { storage: self.storage, reporter: self.reporter, n_http_retries: self.n_http_retries, - n_query_retries: self.n_query_retries, consent_cookie: format!( "{}={}{}", CONSENT_COOKIE, @@ -388,16 +384,6 @@ impl RustyPipeBuilder { self } - /// Set the number of retries for YouTube API queries. - /// - /// If a YouTube API requests returns invalid data, the request is repeated. - /// - /// **Default value**: 1 - pub fn n_query_retries(mut self, n_retries: u32) -> Self { - self.n_query_retries = n_retries; - self - } - /// Set the user agent used for making requests to the web API. /// /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0` @@ -974,62 +960,6 @@ impl RustyPipeQuery { endpoint: &str, body: &B, deobf: Option<&Deobfuscator>, - ) -> Result { - for n in 0..self.client.inner.n_query_retries { - let res = self - ._try_execute_request_deobf::( - ctype, - operation, - id, - endpoint, - body, - deobf, - n == 0, - ) - .await; - let emsg = match res { - Ok(res) => return Ok(res), - Err(error) => match &error { - Error::Extraction(e) => match e { - ExtractionError::Deserialization(_) - | ExtractionError::InvalidData(_) - | ExtractionError::WrongResult(_) - | ExtractionError::Retry => e.to_string(), - _ => return Err(error), - }, - _ => return Err(error), - }, - }; - - warn!("{} retry attempt #{}. Error: {}.", operation, n, emsg); - } - self._try_execute_request_deobf::( - ctype, - operation, - id, - endpoint, - body, - deobf, - self.client.inner.n_query_retries < 2, - ) - .await - } - - /// Single try of `execute_request_deobf` - #[allow(clippy::too_many_arguments)] - async fn _try_execute_request_deobf< - R: DeserializeOwned + MapResponse + Debug, - M, - B: Serialize + ?Sized, - >( - &self, - ctype: ClientType, - operation: &str, - id: &str, - endpoint: &str, - body: &B, - deobf: Option<&Deobfuscator>, - report: bool, ) -> Result { let request = self .request_builder(ctype, endpoint) @@ -1049,32 +979,30 @@ impl RustyPipeQuery { // println!("{}", &resp_str); let create_report = |level: Level, error: Option, msgs: Vec| { - if report { - if let Some(reporter) = &self.client.inner.reporter { - let report = Report { - info: Default::default(), - level, - operation: format!("{}({})", operation, id), - error, - msgs, - deobf_data: deobf.map(Deobfuscator::get_data), - http_request: crate::report::HTTPRequest { - url: request_url, - method: "POST".to_string(), - req_header: request_headers - .iter() - .map(|(k, v)| { - (k.to_string(), v.to_str().unwrap_or_default().to_owned()) - }) - .collect(), - req_body: serde_json::to_string(body).unwrap_or_default(), - status: status.into(), - resp_body: resp_str.to_owned(), - }, - }; + if let Some(reporter) = &self.client.inner.reporter { + let report = Report { + info: Default::default(), + level, + operation: format!("{}({})", operation, id), + error, + msgs, + deobf_data: deobf.map(Deobfuscator::get_data), + http_request: crate::report::HTTPRequest { + url: request_url, + method: "POST".to_string(), + req_header: request_headers + .iter() + .map(|(k, v)| { + (k.to_string(), v.to_str().unwrap_or_default().to_owned()) + }) + .collect(), + req_body: serde_json::to_string(body).unwrap_or_default(), + status: status.into(), + resp_body: resp_str.to_owned(), + }, + }; - reporter.report(&report); - } + reporter.report(&report); } }; @@ -1115,8 +1043,7 @@ impl RustyPipeQuery { match e { ExtractionError::VideoUnavailable(_, _) | ExtractionError::VideoAgeRestricted - | ExtractionError::ContentUnavailable(_) - | ExtractionError::Retry => (), + | ExtractionError::ContentUnavailable(_) => (), _ => create_report(Level::ERR, Some(e.to_string()), Vec::new()), } Err(e.into()) diff --git a/src/client/pagination.rs b/src/client/pagination.rs index ea19bb0..bcfa726 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -51,9 +51,7 @@ impl> MapResponse> for response::Continuati lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { - let mut actions = self - .on_response_received_actions - .ok_or(ExtractionError::Retry)?; + let mut actions = self.on_response_received_actions; let items = some_or_bail!( actions.try_swap_remove(0), Err(ExtractionError::InvalidData( @@ -219,7 +217,6 @@ mod tests { use crate::{ client::{response, MapResponse}, - error::ExtractionError, model::{Paginator, PlaylistItem, YouTubeItem}, param::Language, serializer::MapResult, @@ -268,19 +265,4 @@ mod tests { ); insta::assert_ron_snapshot!(format!("map_{}", name), map_res.c); } - - #[test] - fn map_recommendations_empty() { - let filename = format!("testfiles/video_details/recommendations_empty.json"); - let json_path = Path::new(&filename); - let json_file = File::open(json_path).unwrap(); - - let recommendations: response::Continuation = - serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: Result>, ExtractionError> = - recommendations.map_response("", Language::En, None); - let err = map_res.unwrap_err(); - - assert!(matches!(err, crate::error::ExtractionError::Retry)); - } } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index f30be89..be331fc 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -185,7 +185,9 @@ impl MapResponse> for response::PlaylistCont { _deobf: Option<&Deobfuscator>, ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; - let action = actions.try_swap_remove(0).ok_or(ExtractionError::Retry)?; + let action = actions + .try_swap_remove(0) + .ok_or_else(|| ExtractionError::InvalidData("no onResponseReceivedAction".into()))?; let (items, ctoken) = map_playlist_items(action.append_continuation_items_action.continuation_items.c); diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index aaeea7e..595bf51 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -175,8 +175,8 @@ pub struct Continuation { alias = "onResponseReceivedCommands", alias = "onResponseReceivedEndpoints" )] - #[serde_as(as = "Option>")] - pub on_response_received_actions: Option>, + #[serde_as(as = "VecSkipError<_>")] + pub on_response_received_actions: Vec, } #[derive(Debug, Deserialize)] diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index 06a268b..bca46ed 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -437,8 +437,8 @@ pub struct VideoComments { /// - Comment replies: appendContinuationItemsAction /// - n*commentRenderer, continuationItemRenderer: /// replies + continuation - #[serde_as(as = "Option>")] - pub on_response_received_endpoints: Option>>, + #[serde_as(as = "VecLogError<_>")] + pub on_response_received_endpoints: MapResult>, } /// Video comments continuation diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 0bfa01e..d515f3e 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -346,9 +346,7 @@ impl MapResponse> for response::VideoComments { lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { - let received_endpoints = self - .on_response_received_endpoints - .ok_or(ExtractionError::Retry)?; + let received_endpoints = self.on_response_received_endpoints; let mut warnings = received_endpoints.warnings; let mut comments = Vec::new(); diff --git a/src/error.rs b/src/error.rs index 212c144..23cb14a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -83,6 +83,4 @@ pub enum ExtractionError { WrongResult(String), #[error("Warnings during deserialization/mapping")] DeserializationWarnings, - #[error("Got no data from YouTube, attempt retry")] - Retry, } diff --git a/testfiles/video_details/recommendations_empty.json b/testfiles/video_details/recommendations_empty.json deleted file mode 100644 index d7d28fe..0000000 --- a/testfiles/video_details/recommendations_empty.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "responseContext": { - "visitorData": "CgthSmp5T24zQkRjTSiom5WaBg%3D%3D", - "serviceTrackingParams": [ - { - "service": "CSI", - "params": [ - { - "key": "c", - "value": "WEB" - }, - { - "key": "cver", - "value": "2.20221006.09.00" - }, - { - "key": "yt_li", - "value": "0" - }, - { - "key": "GetWatchNext_rid", - "value": "0x8836d1dc393da349" - } - ] - }, - { - "service": "GFEEDBACK", - "params": [ - { - "key": "logged_in", - "value": "0" - }, - { - "key": "e", - "value": "1714258,23804281,23882503,23885487,23918597,23934970,23940248,23946420,23966208,23983296,23986022,23998056,24001373,24002022,24002025,24004644,24007246,24034168,24036948,24077241,24080738,24108448,24120820,24135310,24140247,24152443,24161116,24162920,24164186,24166867,24169501,24181174,24185614,24187043,24187377,24191629,24197450,24199724,24199774,24211178,24217535,24219713,24223903,24224266,24225483,24226335,24227844,24228638,24229161,24241378,24243988,24248092,24248385,24254502,24255543,24255545,24256985,24259938,24260783,24262346,24263796,24265820,24267564,24267570,24268142,24268812,24268870,24278546,24278596,24279196,24279628,24279727,24280997,24281835,24282957,24283093,24283280,24286003,24286019,24287326,24287795,24288045,24289478,24289901,24289939,24290131,24290276,24290971,24292296,24295099,24295740,24297099,24298640,24298651,24298795,24299688,24299747,24390674,24391537,24392058,24392269,24394618,24590921,39322278,39322399,39322505" - } - ] - }, - { - "service": "GUIDED_HELP", - "params": [ - { - "key": "logged_in", - "value": "0" - } - ] - }, - { - "service": "ECATCHER", - "params": [ - { - "key": "client.version", - "value": "2.20221006" - }, - { - "key": "client.name", - "value": "WEB" - }, - { - "key": "client.fexp", - "value": "24286003,24298795,24256985,23804281,24001373,23946420,24283280,24289478,24223903,24298651,24286019,23885487,24077241,24265820,23918597,24255545,24036948,24259938,24279196,24199774,24282957,24279628,24268812,24169501,24225483,24590921,24197450,24298640,39322399,24290971,24108448,24287326,24219713,24278596,24002022,24181174,24227844,24287795,24229161,24283093,24162920,24248092,24241378,24166867,24002025,24280997,24391537,24278546,24288045,24034168,24290131,24211178,24289901,24226335,24268870,24295099,24135310,24191629,24394618,24007246,24004644,24243988,24281835,24392058,23998056,24185614,24262346,24187043,24224266,23986022,24228638,23934970,39322278,24292296,24260783,23940248,24263796,24267564,24299688,24390674,24152443,23966208,24267570,24080738,24290276,24217535,23882503,24279727,24164186,24289939,24187377,24268142,24120820,24199724,39322505,24392269,24254502,24255543,24299747,24161116,24140247,1714258,24297099,23983296,24295740,24248385" - } - ] - } - ], - "mainAppWebResponseContext": { - "loggedOut": true - }, - "webResponseContextExtensionData": { - "hasDecorated": true - } - }, - "trackingParams": "CAAQg2ciEwjfruPhg9j6AhXW2BEIHd9wAwc=", - "engagementPanels": [ - { - "engagementPanelSectionListRenderer": { - "content": { - "adsEngagementPanelContentRenderer": { - "hack": true - } - }, - "targetId": "engagement-panel-ads", - "visibility": "ENGAGEMENT_PANEL_VISIBILITY_HIDDEN", - "loggingDirectives": { - "trackingParams": "CAEQ040EGAAiEwjfruPhg9j6AhXW2BEIHd9wAwc=", - "visibility": { - "types": "12" - }, - "enableDisplayloggerExperiment": true - } - } - } - ] -}