Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: remove query retries

Browse source
This commit is contained in:
ThetaDev 2022-10-17 23:42:10 +02:00
parent 3a0db09e23
commit ae91c46fb2
8 changed files with 33 additions and 220 deletions
Download patch file Download diff file Expand all files Collapse all files

121
src/client/mod.rs
View file

@ -182,7 +182,6 @@ struct RustyPipeRef {
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
n_http_retries: u32,
n_query_retries: u32,
consent_cookie: String,
cache: CacheHolder,
default_opts: RustyPipeOpts,
@ -200,7 +199,6 @@ pub struct RustyPipeBuilder {
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
n_http_retries: u32,
n_query_retries: u32,
user_agent: String,
default_opts: RustyPipeOpts,
}
@ -292,7 +290,6 @@ impl RustyPipeBuilder {
storage: Some(Box::new(FileStorage::default())),
reporter: Some(Box::new(FileReporter::default())),
n_http_retries: 2,
n_query_retries: 1,
user_agent: DEFAULT_UA.to_owned(),
}
}
@ -328,7 +325,6 @@ impl RustyPipeBuilder {
storage: self.storage,
reporter: self.reporter,
n_http_retries: self.n_http_retries,
n_query_retries: self.n_query_retries,
consent_cookie: format!(
"{}={}{}",
CONSENT_COOKIE,
@ -388,16 +384,6 @@ impl RustyPipeBuilder {
self
}
/// Set the number of retries for YouTube API queries.
///
/// If a YouTube API requests returns invalid data, the request is repeated.
///
/// **Default value**: 1
pub fn n_query_retries(mut self, n_retries: u32) -> Self {
self.n_query_retries = n_retries;
self
}
/// Set the user agent used for making requests to the web API.
///
/// **Default value**: `Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0`
@ -974,62 +960,6 @@ impl RustyPipeQuery {
endpoint: &str,
body: &B,
deobf: Option<&Deobfuscator>,
) -> Result<M, Error> {
for n in 0..self.client.inner.n_query_retries {
let res = self
._try_execute_request_deobf::<R, M, B>(
ctype,
operation,
id,
endpoint,
body,
deobf,
n == 0,
)
.await;
let emsg = match res {
Ok(res) => return Ok(res),
Err(error) => match &error {
Error::Extraction(e) => match e {
ExtractionError::Deserialization(_)
| ExtractionError::InvalidData(_)
| ExtractionError::WrongResult(_)
| ExtractionError::Retry => e.to_string(),
_ => return Err(error),
},
_ => return Err(error),
},
};
warn!("{} retry attempt #{}. Error: {}.", operation, n, emsg);
}
self._try_execute_request_deobf::<R, M, B>(
ctype,
operation,
id,
endpoint,
body,
deobf,
self.client.inner.n_query_retries < 2,
)
.await
}
/// Single try of `execute_request_deobf`
#[allow(clippy::too_many_arguments)]
async fn _try_execute_request_deobf<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
B: Serialize + ?Sized,
>(
&self,
ctype: ClientType,
operation: &str,
id: &str,
endpoint: &str,
body: &B,
deobf: Option<&Deobfuscator>,
report: bool,
) -> Result<M, Error> {
let request = self
.request_builder(ctype, endpoint)
@ -1049,32 +979,30 @@ impl RustyPipeQuery {
// println!("{}", &resp_str);
let create_report = |level: Level, error: Option<String>, msgs: Vec<String>| {
if report {
if let Some(reporter) = &self.client.inner.reporter {
let report = Report {
info: Default::default(),
level,
operation: format!("{}({})", operation, id),
error,
msgs,
deobf_data: deobf.map(Deobfuscator::get_data),
http_request: crate::report::HTTPRequest {
url: request_url,
method: "POST".to_string(),
req_header: request_headers
.iter()
.map(|(k, v)| {
(k.to_string(), v.to_str().unwrap_or_default().to_owned())
})
.collect(),
req_body: serde_json::to_string(body).unwrap_or_default(),
status: status.into(),
resp_body: resp_str.to_owned(),
},
};
if let Some(reporter) = &self.client.inner.reporter {
let report = Report {
info: Default::default(),
level,
operation: format!("{}({})", operation, id),
error,
msgs,
deobf_data: deobf.map(Deobfuscator::get_data),
http_request: crate::report::HTTPRequest {
url: request_url,
method: "POST".to_string(),
req_header: request_headers
.iter()
.map(|(k, v)| {
(k.to_string(), v.to_str().unwrap_or_default().to_owned())
})
.collect(),
req_body: serde_json::to_string(body).unwrap_or_default(),
status: status.into(),
resp_body: resp_str.to_owned(),
},
};
reporter.report(&report);
}
reporter.report(&report);
}
};
@ -1115,8 +1043,7 @@ impl RustyPipeQuery {
match e {
ExtractionError::VideoUnavailable(_, _)
| ExtractionError::VideoAgeRestricted
| ExtractionError::ContentUnavailable(_)
| ExtractionError::Retry => (),
| ExtractionError::ContentUnavailable(_) => (),
_ => create_report(Level::ERR, Some(e.to_string()), Vec::new()),
}
Err(e.into())

20
src/client/pagination.rs
View file

@ -51,9 +51,7 @@ impl<T: TryFrom<YouTubeItem>> MapResponse<Paginator<T>> for response::Continuati
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<T>>, ExtractionError> {
let mut actions = self
.on_response_received_actions
.ok_or(ExtractionError::Retry)?;
let mut actions = self.on_response_received_actions;
let items = some_or_bail!(
actions.try_swap_remove(0),
Err(ExtractionError::InvalidData(
@ -219,7 +217,6 @@ mod tests {
use crate::{
client::{response, MapResponse},
error::ExtractionError,
model::{Paginator, PlaylistItem, YouTubeItem},
param::Language,
serializer::MapResult,
@ -268,19 +265,4 @@ mod tests {
);
insta::assert_ron_snapshot!(format!("map_{}", name), map_res.c);
}
#[test]
fn map_recommendations_empty() {
let filename = format!("testfiles/video_details/recommendations_empty.json");
let json_path = Path::new(&filename);
let json_file = File::open(json_path).unwrap();
let recommendations: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> =
recommendations.map_response("", Language::En, None);
let err = map_res.unwrap_err();
assert!(matches!(err, crate::error::ExtractionError::Retry));
}
}

4
src/client/playlist.rs
View file

@ -185,7 +185,9 @@ impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
_deobf: Option<&Deobfuscator>,
) -> Result<MapResult<Paginator<PlaylistVideo>>, ExtractionError> {
let mut actions = self.on_response_received_actions;
let action = actions.try_swap_remove(0).ok_or(ExtractionError::Retry)?;
let action = actions
.try_swap_remove(0)
.ok_or_else(|| ExtractionError::InvalidData("no onResponseReceivedAction".into()))?;
let (items, ctoken) =
map_playlist_items(action.append_continuation_items_action.continuation_items.c);

4
src/client/response/mod.rs
View file

@ -175,8 +175,8 @@ pub struct Continuation {
alias = "onResponseReceivedCommands",
alias = "onResponseReceivedEndpoints"
)]
#[serde_as(as = "Option<VecSkipError<_>>")]
pub on_response_received_actions: Option<Vec<ContinuationActionWrap>>,
#[serde_as(as = "VecSkipError<_>")]
pub on_response_received_actions: Vec<ContinuationActionWrap>,
}
#[derive(Debug, Deserialize)]

4
src/client/response/video_details.rs
View file

@ -437,8 +437,8 @@ pub struct VideoComments {
/// - Comment replies: appendContinuationItemsAction
/// - n*commentRenderer, continuationItemRenderer:
/// replies + continuation
#[serde_as(as = "Option<VecLogError<_>>")]
pub on_response_received_endpoints: Option<MapResult<Vec<CommentsContItem>>>,
#[serde_as(as = "VecLogError<_>")]
pub on_response_received_endpoints: MapResult<Vec<CommentsContItem>>,
}
/// Video comments continuation

4
src/client/video_details.rs
View file

@ -346,9 +346,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
let received_endpoints = self
.on_response_received_endpoints
.ok_or(ExtractionError::Retry)?;
let received_endpoints = self.on_response_received_endpoints;
let mut warnings = received_endpoints.warnings;
let mut comments = Vec::new();

2
src/error.rs
View file

@ -83,6 +83,4 @@ pub enum ExtractionError {
WrongResult(String),
#[error("Warnings during deserialization/mapping")]
DeserializationWarnings,
#[error("Got no data from YouTube, attempt retry")]
Retry,
}