feat: add HTTP request timeout
This commit is contained in:
parent
c021496a55
commit
a51e42f563
1 changed files with 58 additions and 41 deletions
|
|
@ -23,14 +23,14 @@ mod video_details;
|
||||||
mod channel_rss;
|
mod channel_rss;
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::{borrow::Cow, fmt::Debug};
|
use std::{borrow::Cow, fmt::Debug, time::Duration};
|
||||||
|
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode};
|
use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode};
|
||||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||||
use time::{Duration, OffsetDateTime};
|
use time::OffsetDateTime;
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
|
@ -241,15 +241,30 @@ struct RustyPipeOpts {
|
||||||
|
|
||||||
/// Builder to construct a new RustyPipe client
|
/// Builder to construct a new RustyPipe client
|
||||||
pub struct RustyPipeBuilder {
|
pub struct RustyPipeBuilder {
|
||||||
storage: Option<Box<dyn CacheStorage>>,
|
storage: DefaultOpt<Box<dyn CacheStorage>>,
|
||||||
no_storage: bool,
|
reporter: DefaultOpt<Box<dyn Reporter>>,
|
||||||
reporter: Option<Box<dyn Reporter>>,
|
|
||||||
no_reporter: bool,
|
|
||||||
n_http_retries: u32,
|
n_http_retries: u32,
|
||||||
|
timeout: DefaultOpt<Duration>,
|
||||||
user_agent: Option<String>,
|
user_agent: Option<String>,
|
||||||
default_opts: RustyPipeOpts,
|
default_opts: RustyPipeOpts,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum DefaultOpt<T> {
|
||||||
|
Some(T),
|
||||||
|
None,
|
||||||
|
Default,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<T> DefaultOpt<T> {
|
||||||
|
fn or_default<F: FnOnce() -> T>(self, f: F) -> Option<T> {
|
||||||
|
match self {
|
||||||
|
DefaultOpt::Some(x) => Some(x),
|
||||||
|
DefaultOpt::None => None,
|
||||||
|
DefaultOpt::Default => Some(f()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// RustyPipe query object
|
/// RustyPipe query object
|
||||||
///
|
///
|
||||||
/// Contains a reference to the RustyPipe client as well as query-specific
|
/// Contains a reference to the RustyPipe client as well as query-specific
|
||||||
|
|
@ -308,7 +323,7 @@ impl<T> CacheEntry<T> {
|
||||||
fn get(&self) -> Option<&T> {
|
fn get(&self) -> Option<&T> {
|
||||||
match self {
|
match self {
|
||||||
CacheEntry::Some { last_update, data } => {
|
CacheEntry::Some { last_update, data } => {
|
||||||
if last_update < &(OffsetDateTime::now_utc() - Duration::hours(24)) {
|
if last_update < &(OffsetDateTime::now_utc() - time::Duration::hours(24)) {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
Some(data)
|
Some(data)
|
||||||
|
|
@ -341,10 +356,9 @@ impl RustyPipeBuilder {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
RustyPipeBuilder {
|
RustyPipeBuilder {
|
||||||
default_opts: RustyPipeOpts::default(),
|
default_opts: RustyPipeOpts::default(),
|
||||||
storage: None,
|
storage: DefaultOpt::Default,
|
||||||
no_storage: false,
|
reporter: DefaultOpt::Default,
|
||||||
reporter: None,
|
timeout: DefaultOpt::Default,
|
||||||
no_reporter: false,
|
|
||||||
n_http_retries: 2,
|
n_http_retries: 2,
|
||||||
user_agent: None,
|
user_agent: None,
|
||||||
}
|
}
|
||||||
|
|
@ -352,15 +366,19 @@ impl RustyPipeBuilder {
|
||||||
|
|
||||||
/// Returns a new, configured RustyPipe instance.
|
/// Returns a new, configured RustyPipe instance.
|
||||||
pub fn build(self) -> RustyPipe {
|
pub fn build(self) -> RustyPipe {
|
||||||
let http = ClientBuilder::new()
|
let mut client_builder = ClientBuilder::new()
|
||||||
.user_agent(self.user_agent.unwrap_or_else(|| DEFAULT_UA.to_owned()))
|
.user_agent(self.user_agent.unwrap_or_else(|| DEFAULT_UA.to_owned()))
|
||||||
.gzip(true)
|
.gzip(true)
|
||||||
.brotli(true)
|
.brotli(true)
|
||||||
.redirect(reqwest::redirect::Policy::none())
|
.redirect(reqwest::redirect::Policy::none());
|
||||||
.build()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let cdata = if let Some(storage) = &self.storage {
|
if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(10)) {
|
||||||
|
client_builder = client_builder.timeout(timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
let http = client_builder.build().unwrap();
|
||||||
|
|
||||||
|
let cdata = if let DefaultOpt::Some(storage) = &self.storage {
|
||||||
if let Some(data) = storage.read() {
|
if let Some(data) = storage.read() {
|
||||||
match serde_json::from_str::<CacheData>(&data) {
|
match serde_json::from_str::<CacheData>(&data) {
|
||||||
Ok(data) => data,
|
Ok(data) => data,
|
||||||
|
|
@ -379,22 +397,8 @@ impl RustyPipeBuilder {
|
||||||
RustyPipe {
|
RustyPipe {
|
||||||
inner: Arc::new(RustyPipeRef {
|
inner: Arc::new(RustyPipeRef {
|
||||||
http,
|
http,
|
||||||
storage: if self.no_storage {
|
storage: self.storage.or_default(|| Box::<FileStorage>::default()),
|
||||||
None
|
reporter: self.reporter.or_default(|| Box::<FileReporter>::default()),
|
||||||
} else {
|
|
||||||
Some(
|
|
||||||
self.storage
|
|
||||||
.unwrap_or_else(|| Box::<FileStorage>::default()),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
reporter: if self.no_reporter {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(
|
|
||||||
self.reporter
|
|
||||||
.unwrap_or_else(|| Box::<FileReporter>::default()),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
n_http_retries: self.n_http_retries,
|
n_http_retries: self.n_http_retries,
|
||||||
consent_cookie: format!(
|
consent_cookie: format!(
|
||||||
"{}={}{}",
|
"{}={}{}",
|
||||||
|
|
@ -418,15 +422,13 @@ impl RustyPipeBuilder {
|
||||||
///
|
///
|
||||||
/// **Default value**: [`FileStorage`] in `rustypipe_cache.json`
|
/// **Default value**: [`FileStorage`] in `rustypipe_cache.json`
|
||||||
pub fn storage(mut self, storage: Box<dyn CacheStorage>) -> Self {
|
pub fn storage(mut self, storage: Box<dyn CacheStorage>) -> Self {
|
||||||
self.storage = Some(storage);
|
self.storage = DefaultOpt::Some(storage);
|
||||||
self.no_storage = false;
|
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Disable cache storage
|
/// Disable cache storage
|
||||||
pub fn no_storage(mut self) -> Self {
|
pub fn no_storage(mut self) -> Self {
|
||||||
self.storage = None;
|
self.storage = DefaultOpt::None;
|
||||||
self.no_storage = true;
|
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -434,15 +436,30 @@ impl RustyPipeBuilder {
|
||||||
///
|
///
|
||||||
/// **Default value**: [`FileReporter`] creating reports in `./rustypipe_reports`
|
/// **Default value**: [`FileReporter`] creating reports in `./rustypipe_reports`
|
||||||
pub fn reporter(mut self, reporter: Box<dyn Reporter>) -> Self {
|
pub fn reporter(mut self, reporter: Box<dyn Reporter>) -> Self {
|
||||||
self.reporter = Some(reporter);
|
self.reporter = DefaultOpt::Some(reporter);
|
||||||
self.no_reporter = false;
|
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Disable the creation of report files in case of errors and warnings.
|
/// Disable the creation of report files in case of errors and warnings.
|
||||||
pub fn no_reporter(mut self) -> Self {
|
pub fn no_reporter(mut self) -> Self {
|
||||||
self.reporter = None;
|
self.reporter = DefaultOpt::None;
|
||||||
self.no_reporter = true;
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Enable a HTTP request timeout
|
||||||
|
///
|
||||||
|
/// The timeout is applied from when the request starts connecting until the
|
||||||
|
/// response body has finished.
|
||||||
|
///
|
||||||
|
/// **Default value**: 10s
|
||||||
|
pub fn timeout(mut self, timeout: Duration) -> Self {
|
||||||
|
self.timeout = DefaultOpt::Some(timeout);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Disable the HTTP request timeout.
|
||||||
|
pub fn no_timeout(mut self) -> Self {
|
||||||
|
self.timeout = DefaultOpt::None;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -576,7 +593,7 @@ impl RustyPipe {
|
||||||
|
|
||||||
let ms = util::retry_delay(n, 1000, 60000, 3);
|
let ms = util::retry_delay(n, 1000, 60000, 3);
|
||||||
log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
|
log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
|
tokio::time::sleep(Duration::from_millis(ms.into())).await;
|
||||||
|
|
||||||
last_res = Some(res);
|
last_res = Some(res);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Reference in a new issue