refactor: remove query retries

2022-10-17 23:42:10 +02:00 · 2022-10-17 23:42:10 +02:00 · ae91c46fb2
commit ae91c46fb2
parent 3a0db09e23
8 changed files with 33 additions and 220 deletions
--- a/src/client/mod.rs
+++ b/src/client/mod.rs
@ -182,7 +182,6 @@ struct RustyPipeRef {
    storage: Option<Box<dyn CacheStorage>>,
    reporter: Option<Box<dyn Reporter>>,
    n_http_retries: u32,
-    n_query_retries: u32,
    consent_cookie: String,
    cache: CacheHolder,
    default_opts: RustyPipeOpts,
@ -200,7 +199,6 @@ pub struct RustyPipeBuilder {
    storage: Option<Box<dyn CacheStorage>>,
    reporter: Option<Box<dyn Reporter>>,
    n_http_retries: u32,
-    n_query_retries: u32,
    user_agent: String,
    default_opts: RustyPipeOpts,
 }
@ -292,7 +290,6 @@ impl RustyPipeBuilder {
            storage: Some(Box::new(FileStorage::default())),
            reporter: Some(Box::new(FileReporter::default())),
            n_http_retries: 2,
-            n_query_retries: 1,
            user_agent: DEFAULT_UA.to_owned(),
        }
    }
@ -328,7 +325,6 @@ impl RustyPipeBuilder {
                storage: self.storage,
                reporter: self.reporter,
                n_http_retries: self.n_http_retries,
-                n_query_retries: self.n_query_retries,
                consent_cookie: format!(
                    "{}={}{}",
                    CONSENT_COOKIE,
@ -388,16 +384,6 @@ impl RustyPipeBuilder {
        self
    }

-    /// Set the number of retries for YouTube API queries.
-    ///
-    /// If a YouTube API requests returns invalid data, the request is repeated.
-    ///
-    /// **Default value**: 1
-    pub fn n_query_retries(mut self, n_retries: u32) -> Self {
-        self.n_query_retries = n_retries;
-        self
-    }
-
    /// Set the user agent used for making requests to the web API.
    ///
    /// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0`
@ -974,62 +960,6 @@ impl RustyPipeQuery {
        endpoint: &str,
        body: &B,
        deobf: Option<&Deobfuscator>,
-    ) -> Result<M, Error> {
-        for n in 0..self.client.inner.n_query_retries {
-            let res = self
-                ._try_execute_request_deobf::<R, M, B>(
-                    ctype,
-                    operation,
-                    id,
-                    endpoint,
-                    body,
-                    deobf,
-                    n == 0,
-                )
-                .await;
-            let emsg = match res {
-                Ok(res) => return Ok(res),
-                Err(error) => match &error {
-                    Error::Extraction(e) => match e {
-                        ExtractionError::Deserialization(_)
-                        | ExtractionError::InvalidData(_)
-                        | ExtractionError::WrongResult(_)
-                        | ExtractionError::Retry => e.to_string(),
-                        _ => return Err(error),
-                    },
-                    _ => return Err(error),
-                },
-            };
-
-            warn!("{} retry attempt #{}. Error: {}.", operation, n, emsg);
-        }
-        self._try_execute_request_deobf::<R, M, B>(
-            ctype,
-            operation,
-            id,
-            endpoint,
-            body,
-            deobf,
-            self.client.inner.n_query_retries < 2,
-        )
-        .await
-    }
-
-    /// Single try of `execute_request_deobf`
-    #[allow(clippy::too_many_arguments)]
-    async fn _try_execute_request_deobf<
-        R: DeserializeOwned + MapResponse<M> + Debug,
-        M,
-        B: Serialize + ?Sized,
-    >(
-        &self,
-        ctype: ClientType,
-        operation: &str,
-        id: &str,
-        endpoint: &str,
-        body: &B,
-        deobf: Option<&Deobfuscator>,
-        report: bool,
    ) -> Result<M, Error> {
        let request = self
            .request_builder(ctype, endpoint)
@ -1049,32 +979,30 @@ impl RustyPipeQuery {
        // println!("{}", &resp_str);

        let create_report = |level: Level, error: Option<String>, msgs: Vec<String>| {
-            if report {
-                if let Some(reporter) = &self.client.inner.reporter {
-                    let report = Report {
-                        info: Default::default(),
-                        level,
-                        operation: format!("{}({})", operation, id),
-                        error,
-                        msgs,
-                        deobf_data: deobf.map(Deobfuscator::get_data),
-                        http_request: crate::report::HTTPRequest {
-                            url: request_url,
-                            method: "POST".to_string(),
-                            req_header: request_headers
-                                .iter()
-                                .map(|(k, v)| {
-                                    (k.to_string(), v.to_str().unwrap_or_default().to_owned())
-                                })
-                                .collect(),
-                            req_body: serde_json::to_string(body).unwrap_or_default(),
-                            status: status.into(),
-                            resp_body: resp_str.to_owned(),
-                        },
-                    };
+            if let Some(reporter) = &self.client.inner.reporter {
+                let report = Report {
+                    info: Default::default(),
+                    level,
+                    operation: format!("{}({})", operation, id),
+                    error,
+                    msgs,
+                    deobf_data: deobf.map(Deobfuscator::get_data),
+                    http_request: crate::report::HTTPRequest {
+                        url: request_url,
+                        method: "POST".to_string(),
+                        req_header: request_headers
+                            .iter()
+                            .map(|(k, v)| {
+                                (k.to_string(), v.to_str().unwrap_or_default().to_owned())
+                            })
+                            .collect(),
+                        req_body: serde_json::to_string(body).unwrap_or_default(),
+                        status: status.into(),
+                        resp_body: resp_str.to_owned(),
+                    },
+                };

-                    reporter.report(&report);
-                }
+                reporter.report(&report);
            }
        };

@ -1115,8 +1043,7 @@ impl RustyPipeQuery {
                    match e {
                        ExtractionError::VideoUnavailable(_, _)
                        | ExtractionError::VideoAgeRestricted
-                        | ExtractionError::ContentUnavailable(_)
-                        | ExtractionError::Retry => (),
+                        | ExtractionError::ContentUnavailable(_) => (),
                        _ => create_report(Level::ERR, Some(e.to_string()), Vec::new()),
                    }
                    Err(e.into())
--- a/src/client/pagination.rs
+++ b/src/client/pagination.rs
@ -51,9 +51,7 @@ impl<T: TryFrom<YouTubeItem>> MapResponse<Paginator<T>> for response::Continuati
        lang: crate::param::Language,
        _deobf: Option<&crate::deobfuscate::Deobfuscator>,
    ) -> Result<MapResult<Paginator<T>>, ExtractionError> {
-        let mut actions = self
-            .on_response_received_actions
-            .ok_or(ExtractionError::Retry)?;
+        let mut actions = self.on_response_received_actions;
        let items = some_or_bail!(
            actions.try_swap_remove(0),
            Err(ExtractionError::InvalidData(
@ -219,7 +217,6 @@ mod tests {

    use crate::{
        client::{response, MapResponse},
-        error::ExtractionError,
        model::{Paginator, PlaylistItem, YouTubeItem},
        param::Language,
        serializer::MapResult,
@ -268,19 +265,4 @@ mod tests {
        );
        insta::assert_ron_snapshot!(format!("map_{}", name), map_res.c);
    }
-
-    #[test]
-    fn map_recommendations_empty() {
-        let filename = format!("testfiles/video_details/recommendations_empty.json");
-        let json_path = Path::new(&filename);
-        let json_file = File::open(json_path).unwrap();
-
-        let recommendations: response::Continuation =
-            serde_json::from_reader(BufReader::new(json_file)).unwrap();
-        let map_res: Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> =
-            recommendations.map_response("", Language::En, None);
-        let err = map_res.unwrap_err();
-
-        assert!(matches!(err, crate::error::ExtractionError::Retry));
-    }
 }
--- a/src/client/playlist.rs
+++ b/src/client/playlist.rs
@ -185,7 +185,9 @@ impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
        _deobf: Option<&Deobfuscator>,
    ) -> Result<MapResult<Paginator<PlaylistVideo>>, ExtractionError> {
        let mut actions = self.on_response_received_actions;
-        let action = actions.try_swap_remove(0).ok_or(ExtractionError::Retry)?;
+        let action = actions
+            .try_swap_remove(0)
+            .ok_or_else(|| ExtractionError::InvalidData("no onResponseReceivedAction".into()))?;

        let (items, ctoken) =
            map_playlist_items(action.append_continuation_items_action.continuation_items.c);
--- a/src/client/response/mod.rs
+++ b/src/client/response/mod.rs
@ -175,8 +175,8 @@ pub struct Continuation {
        alias = "onResponseReceivedCommands",
        alias = "onResponseReceivedEndpoints"
    )]
-    #[serde_as(as = "Option<VecSkipError<_>>")]
-    pub on_response_received_actions: Option<Vec<ContinuationActionWrap>>,
+    #[serde_as(as = "VecSkipError<_>")]
+    pub on_response_received_actions: Vec<ContinuationActionWrap>,
 }

 #[derive(Debug, Deserialize)]
--- a/src/client/response/video_details.rs
+++ b/src/client/response/video_details.rs
@ -437,8 +437,8 @@ pub struct VideoComments {
    /// - Comment replies: appendContinuationItemsAction
    ///   - n*commentRenderer, continuationItemRenderer:
    ///     replies + continuation
-    #[serde_as(as = "Option<VecLogError<_>>")]
-    pub on_response_received_endpoints: Option<MapResult<Vec<CommentsContItem>>>,
+    #[serde_as(as = "VecLogError<_>")]
+    pub on_response_received_endpoints: MapResult<Vec<CommentsContItem>>,
 }

 /// Video comments continuation
--- a/src/client/video_details.rs
+++ b/src/client/video_details.rs
@ -346,9 +346,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
        lang: Language,
        _deobf: Option<&crate::deobfuscate::Deobfuscator>,
    ) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
-        let received_endpoints = self
-            .on_response_received_endpoints
-            .ok_or(ExtractionError::Retry)?;
+        let received_endpoints = self.on_response_received_endpoints;
        let mut warnings = received_endpoints.warnings;

        let mut comments = Vec::new();
--- a/src/error.rs
+++ b/src/error.rs
@ -83,6 +83,4 @@ pub enum ExtractionError {
    WrongResult(String),
    #[error("Warnings during deserialization/mapping")]
    DeserializationWarnings,
-    #[error("Got no data from YouTube, attempt retry")]
-    Retry,
 }
--- a/testfiles/video_details/recommendations_empty.json
+++ b/testfiles/video_details/recommendations_empty.json
@ -1,94 +0,0 @@
-{
-  "responseContext": {
-    "visitorData": "CgthSmp5T24zQkRjTSiom5WaBg%3D%3D",
-    "serviceTrackingParams": [
-      {
-        "service": "CSI",
-        "params": [
-          {
-            "key": "c",
-            "value": "WEB"
-          },
-          {
-            "key": "cver",
-            "value": "2.20221006.09.00"
-          },
-          {
-            "key": "yt_li",
-            "value": "0"
-          },
-          {
-            "key": "GetWatchNext_rid",
-            "value": "0x8836d1dc393da349"
-          }
-        ]
-      },
-      {
-        "service": "GFEEDBACK",
-        "params": [
-          {
-            "key": "logged_in",
-            "value": "0"
-          },
-          {
-            "key": "e",
-            "value": "1714258,23804281,23882503,23885487,23918597,23934970,23940248,23946420,23966208,23983296,23986022,23998056,24001373,24002022,24002025,24004644,24007246,24034168,24036948,24077241,24080738,24108448,24120820,24135310,24140247,24152443,24161116,24162920,24164186,24166867,24169501,24181174,24185614,24187043,24187377,24191629,24197450,24199724,24199774,24211178,24217535,24219713,24223903,24224266,24225483,24226335,24227844,24228638,24229161,24241378,24243988,24248092,24248385,24254502,24255543,24255545,24256985,24259938,24260783,24262346,24263796,24265820,24267564,24267570,24268142,24268812,24268870,24278546,24278596,24279196,24279628,24279727,24280997,24281835,24282957,24283093,24283280,24286003,24286019,24287326,24287795,24288045,24289478,24289901,24289939,24290131,24290276,24290971,24292296,24295099,24295740,24297099,24298640,24298651,24298795,24299688,24299747,24390674,24391537,24392058,24392269,24394618,24590921,39322278,39322399,39322505"
-          }
-        ]
-      },
-      {
-        "service": "GUIDED_HELP",
-        "params": [
-          {
-            "key": "logged_in",
-            "value": "0"
-          }
-        ]
-      },
-      {
-        "service": "ECATCHER",
-        "params": [
-          {
-            "key": "client.version",
-            "value": "2.20221006"
-          },
-          {
-            "key": "client.name",
-            "value": "WEB"
-          },
-          {
-            "key": "client.fexp",
-            "value": "24286003,24298795,24256985,23804281,24001373,23946420,24283280,24289478,24223903,24298651,24286019,23885487,24077241,24265820,23918597,24255545,24036948,24259938,24279196,24199774,24282957,24279628,24268812,24169501,24225483,24590921,24197450,24298640,39322399,24290971,24108448,24287326,24219713,24278596,24002022,24181174,24227844,24287795,24229161,24283093,24162920,24248092,24241378,24166867,24002025,24280997,24391537,24278546,24288045,24034168,24290131,24211178,24289901,24226335,24268870,24295099,24135310,24191629,24394618,24007246,24004644,24243988,24281835,24392058,23998056,24185614,24262346,24187043,24224266,23986022,24228638,23934970,39322278,24292296,24260783,23940248,24263796,24267564,24299688,24390674,24152443,23966208,24267570,24080738,24290276,24217535,23882503,24279727,24164186,24289939,24187377,24268142,24120820,24199724,39322505,24392269,24254502,24255543,24299747,24161116,24140247,1714258,24297099,23983296,24295740,24248385"
-          }
-        ]
-      }
-    ],
-    "mainAppWebResponseContext": {
-      "loggedOut": true
-    },
-    "webResponseContextExtensionData": {
-      "hasDecorated": true
-    }
-  },
-  "trackingParams": "CAAQg2ciEwjfruPhg9j6AhXW2BEIHd9wAwc=",
-  "engagementPanels": [
-    {
-      "engagementPanelSectionListRenderer": {
-        "content": {
-          "adsEngagementPanelContentRenderer": {
-            "hack": true
-          }
-        },
-        "targetId": "engagement-panel-ads",
-        "visibility": "ENGAGEMENT_PANEL_VISIBILITY_HIDDEN",
-        "loggingDirectives": {
-          "trackingParams": "CAEQ040EGAAiEwjfruPhg9j6AhXW2BEIHd9wAwc=",
-          "visibility": {
-            "types": "12"
-          },
-          "enableDisplayloggerExperiment": true
-        }
-      }
-    }
-  ]
-}