From 3a2370b97ca3d0f40d72d66a23295557317d29fb Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sat, 18 Jan 2025 07:03:36 +0100 Subject: [PATCH] feat: add timezone query option --- Cargo.toml | 1 + cli/Cargo.toml | 3 ++ cli/src/main.rs | 18 +++++++++++ src/client/channel.rs | 6 ++-- src/client/history.rs | 4 +-- src/client/mod.rs | 50 ++++++++++++++++++++++++++++++- src/client/pagination.rs | 5 ++-- src/client/player.rs | 2 ++ src/client/playlist.rs | 11 +++++-- src/client/response/video_item.rs | 21 ++++++++++--- src/client/search.rs | 2 +- src/client/trends.rs | 2 +- src/client/video_details.rs | 9 ++++-- src/util/timeago.rs | 23 +++++++++----- 14 files changed, 132 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index efc8ce3..7d4dd18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ time = { version = "0.3.37", features = [ "serde-human-readable", "serde-well-known", ] } +time-tz = { version = "2.0.0" } futures-util = "0.3.31" ress = "0.11.0" phf = "0.11.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 519d210..a1c2257 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,6 +12,7 @@ description = "CLI for RustyPipe - download videos and extract data from YouTube [features] default = ["native-tls"] +timezone = ["dep:time", "dep:time-tz"] # Reqwest TLS options native-tls = [ @@ -49,6 +50,8 @@ futures-util.workspace = true serde.workspace = true serde_json.workspace = true quick-xml.workspace = true +time = { workspace = true, optional = true } +time-tz = { workspace = true, optional = true } indicatif.workspace = true anyhow.workspace = true diff --git a/cli/src/main.rs b/cli/src/main.rs index 7f49119..68ff919 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -55,6 +55,10 @@ struct Cli { /// YouTube content country #[clap(long, global = true)] country: Option, + /// UTC offset in minutes + #[cfg(feature = "timezone")] + #[clap(long, global = true)] + timezone: Option, /// Use authentication #[clap(long, global = true)] auth: bool, @@ -913,6 +917,20 @@ async fn run() -> anyhow::Result<()> { if let Some(botguard_bin) = cli.botguard_bin { rp = rp.botguard_bin(botguard_bin); } + + #[cfg(feature = "timezone")] + if let Some(timezone) = cli.timezone { + use time::OffsetDateTime; + use time_tz::{Offset, TimeZone}; + + let tz = time_tz::timezones::get_by_name(&timezone).expect("invalid timezone"); + let offset = tz + .get_offset_utc(&OffsetDateTime::now_utc()) + .to_utc() + .whole_minutes(); + rp = rp.timezone(tz.name(), offset); + } + if cli.no_botguard { rp = rp.no_botguard(); } diff --git a/src/client/channel.rs b/src/client/channel.rs index 533e0a3..7945719 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -220,6 +220,7 @@ impl MapResponse>> for response::Channel { let mut mapper = response::YouTubeListMapper::::with_channel( ctx.lang, + ctx.utc_offset, &channel_data.c, channel_data.warnings, ); @@ -265,6 +266,7 @@ impl MapResponse>> for response::Channel { let mut mapper = response::YouTubeListMapper::::with_channel( ctx.lang, + ctx.utc_offset, &channel_data.c, channel_data.warnings, ); @@ -280,7 +282,7 @@ impl MapResponse>> for response::Channel { impl MapResponse for response::ChannelAbout { fn map_response(self, ctx: &MapRespCtx<'_>) -> Result, ExtractionError> { - // Channel info is always fetched in English. There is no localized data there + // Channel info is always fetched in English. There is no localized data // and it allows parsing the country name. let lang = Language::En; @@ -335,7 +337,7 @@ impl MapResponse for response::ChannelAbout { .video_count_text .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), create_date: about.joined_date_text.and_then(|txt| { - timeago::parse_textual_date_or_warn(lang, &txt, &mut warnings) + timeago::parse_textual_date_or_warn(lang, ctx.utc_offset, &txt, &mut warnings) .map(OffsetDateTime::date) }), view_count: about diff --git a/src/client/history.rs b/src/client/history.rs index 6852b60..d0272b2 100644 --- a/src/client/history.rs +++ b/src/client/history.rs @@ -177,7 +177,7 @@ impl MapResponse>> for response::History { for item in items.c { match item { response::YouTubeListItem::ItemSectionRenderer { header, contents } => { - let mut mapper = YouTubeListMapper::::new(ctx.lang); + let mut mapper = YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(contents); mapper.conv_history_items( header.map(|h| h.item_section_header_renderer.title), @@ -228,7 +228,7 @@ impl MapResponse> for response::History { .section_list_renderer .contents; - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(items); Ok(MapResult { diff --git a/src/client/mod.rs b/src/client/mod.rs index 7102f47..a44f346 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -35,7 +35,7 @@ use regex::Regex; use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sha1::{Digest, Sha1}; -use time::OffsetDateTime; +use time::{OffsetDateTime, UtcOffset}; use tokio::sync::RwLock as AsyncRwLock; use crate::error::AuthError; @@ -386,6 +386,8 @@ struct RustyPipeRef { struct RustyPipeOpts { lang: Language, country: Country, + timezone: Option, + utc_offset_minutes: i16, report: bool, strict: bool, auth: Option, @@ -525,6 +527,8 @@ impl Default for RustyPipeOpts { Self { lang: Language::En, country: Country::Us, + timezone: None, + utc_offset_minutes: 0, report: false, strict: false, auth: None, @@ -890,6 +894,21 @@ impl RustyPipeBuilder { self } + /// Set the timezone and its associated UTC offset in minutes used + /// when accessing the YouTube API. + /// + /// This will also change the UTC offset of the returned dates. + /// + /// **Default value**: `0` (UTC) + /// + /// **Info**: you can set this option for individual queries, too + #[must_use] + pub fn timezone>(mut self, timezone: S, utc_offset_minutes: i16) -> Self { + self.default_opts.timezone = Some(timezone.into()); + self.default_opts.utc_offset_minutes = utc_offset_minutes; + self + } + /// Generate a report on every operation. /// /// This should only be used for debugging. @@ -1668,6 +1687,17 @@ impl RustyPipeQuery { self } + /// Set the timezone and its associated UTC offset in minutes used + /// when accessing the YouTube API. + /// + /// This will also change the UTC offset of the returned dates. + #[must_use] + pub fn timezone>(mut self, timezone: S, utc_offset_minutes: i16) -> Self { + self.opts.timezone = Some(timezone.into()); + self.opts.utc_offset_minutes = utc_offset_minutes; + self + } + /// Generate a report on every operation. /// /// This should only be used for debugging. @@ -1822,6 +1852,8 @@ impl RustyPipeQuery { } else { (Language::En, Country::Us) }; + let utc_offset_minutes = self.opts.utc_offset_minutes; + let time_zone = self.opts.timezone.as_deref().unwrap_or("UTC"); match ctype { ClientType::Desktop => YTContext { @@ -1833,6 +1865,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), @@ -1848,6 +1882,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), @@ -1863,6 +1899,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), @@ -1879,6 +1917,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: Some(RequestYT::default()), @@ -1898,6 +1938,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: None, @@ -1915,6 +1957,8 @@ impl RustyPipeQuery { visitor_data, hl, gl, + time_zone, + utc_offset_minutes, ..Default::default() }, request: None, @@ -2212,6 +2256,8 @@ impl RustyPipeQuery { let ctx = MapRespCtx { id, lang: self.opts.lang, + utc_offset: UtcOffset::from_whole_seconds(i32::from(self.opts.utc_offset_minutes) * 60) + .map_err(|_| Error::Other("utc_offset overflow".into()))?, deobf: ctx_src.deobf, visitor_data: Some(&visitor_data), client_type: ctype, @@ -2498,6 +2544,7 @@ impl AsRef for RustyPipeQuery { struct MapRespCtx<'a> { id: &'a str, lang: Language, + utc_offset: UtcOffset, deobf: Option<&'a DeobfData>, visitor_data: Option<&'a str>, client_type: ClientType, @@ -2525,6 +2572,7 @@ impl<'a> MapRespCtx<'a> { Self { id, lang: Language::En, + utc_offset: UtcOffset::UTC, deobf: None, visitor_data: None, client_type: ClientType::Desktop, diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 3fad657..2948d06 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -127,7 +127,7 @@ impl MapResponse> for response::Continuation { let estimated_results = self.estimated_results; let items = continuation_items(self); - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(items); Ok(MapResult { @@ -237,7 +237,8 @@ impl MapResponse>> for response::Continuation { for item in items.c { match item { response::YouTubeListItem::ItemSectionRenderer { header, contents } => { - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = + response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(contents); mapper.conv_history_items( header.map(|h| h.item_section_header_renderer.title), diff --git a/src/client/player.rs b/src/client/player.rs index d85394c..f5e0c53 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -937,6 +937,7 @@ mod tests { use path_macro::path; use rstest::rstest; + use time::UtcOffset; use super::*; use crate::{deobfuscate::DeobfData, param::Language, util::tests::TESTFILES}; @@ -968,6 +969,7 @@ mod tests { .map_response(&MapRespCtx { id: "pPvd8UxmSbQ", lang: Language::En, + utc_offset: UtcOffset::UTC, deobf: Some(&DEOBF_DATA), visitor_data: None, client_type, diff --git a/src/client/playlist.rs b/src/client/playlist.rs index c2872e6..bf6913a 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -90,7 +90,7 @@ impl MapResponse for response::Playlist { .playlist_video_list_renderer .contents; - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(video_items); let (description, thumbnails, last_update_txt) = match self.sidebar { @@ -225,8 +225,13 @@ impl MapResponse for response::Playlist { .as_deref() .or(last_update_txt2.as_deref()) .and_then(|txt| { - timeago::parse_textual_date_or_warn(ctx.lang, txt, &mut mapper.warnings) - .map(OffsetDateTime::date) + timeago::parse_textual_date_or_warn( + ctx.lang, + ctx.utc_offset, + txt, + &mut mapper.warnings, + ) + .map(OffsetDateTime::date) }); Ok(MapResult { diff --git a/src/client/response/video_item.rs b/src/client/response/video_item.rs index 1541728..001d8d1 100644 --- a/src/client/response/video_item.rs +++ b/src/client/response/video_item.rs @@ -2,7 +2,7 @@ use serde::Deserialize; use serde_with::{ rust::deserialize_ignore_any, serde_as, DefaultOnError, DisplayFromStr, VecSkipError, }; -use time::OffsetDateTime; +use time::{OffsetDateTime, UtcOffset}; use super::{ ChannelBadge, ContentImage, ContinuationEndpoint, PhMetadataView, SimpleHeaderRenderer, @@ -461,6 +461,7 @@ impl IsShort for Vec { #[derive(Debug)] pub(crate) struct YouTubeListMapper { lang: Language, + utc_offset: UtcOffset, channel: Option, pub items: Vec, @@ -470,9 +471,10 @@ pub(crate) struct YouTubeListMapper { } impl YouTubeListMapper { - pub fn new(lang: Language) -> Self { + pub fn new(lang: Language, utc_offset: UtcOffset) -> Self { Self { lang, + utc_offset, channel: None, items: Vec::new(), warnings: Vec::new(), @@ -481,9 +483,15 @@ impl YouTubeListMapper { } } - pub fn with_channel(lang: Language, channel: &Channel, warnings: Vec) -> Self { + pub fn with_channel( + lang: Language, + utc_offset: UtcOffset, + channel: &Channel, + warnings: Vec, + ) -> Self { Self { lang, + utc_offset, channel: Some(ChannelTag { id: channel.id.clone(), name: channel.name.clone(), @@ -786,7 +794,12 @@ impl YouTubeListMapper { thumbnail: tn.image.into(), channel, publish_date: publish_date_txt.as_deref().and_then(|t| { - timeago::parse_textual_date_or_warn(self.lang, t, &mut self.warnings) + timeago::parse_textual_date_or_warn( + self.lang, + self.utc_offset, + t, + &mut self.warnings, + ) }), publish_date_txt, view_count, diff --git a/src/client/search.rs b/src/client/search.rs index b4ba544..5ab5793 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -107,7 +107,7 @@ impl MapResponse> for response::Search { .section_list_renderer .contents; - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(items); Ok(MapResult { diff --git a/src/client/trends.rs b/src/client/trends.rs index fa3510c..10e8fe9 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -45,7 +45,7 @@ impl MapResponse> for response::Trending { .section_list_renderer .contents; - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(items); Ok(MapResult { diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 9cc1ffc..f027c12 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -180,7 +180,12 @@ impl MapResponse for response::VideoDetails { // so we ignore parse errors here for now like_text.and_then(|txt| util::parse_numeric(&txt).ok()), date_text.as_deref().and_then(|txt| { - timeago::parse_textual_date_or_warn(ctx.lang, txt, &mut warnings) + timeago::parse_textual_date_or_warn( + ctx.lang, + ctx.utc_offset, + txt, + &mut warnings, + ) }), date_text, view_count @@ -470,7 +475,7 @@ fn map_recommendations( visitor_data: Option, ctx: &MapRespCtx<'_>, ) -> MapResult> { - let mut mapper = response::YouTubeListMapper::::new(ctx.lang); + let mut mapper = response::YouTubeListMapper::::new(ctx.lang, ctx.utc_offset); mapper.map_response(r); mapper.ctoken = mapper.ctoken.or_else(|| { diff --git a/src/util/timeago.rs b/src/util/timeago.rs index 274c653..65b85a7 100644 --- a/src/util/timeago.rs +++ b/src/util/timeago.rs @@ -13,7 +13,7 @@ use std::ops::Mul; use serde::{Deserialize, Serialize}; -use time::{Date, Duration, Month, OffsetDateTime}; +use time::{Date, Duration, Month, OffsetDateTime, UtcOffset}; use crate::{ param::Language, @@ -333,8 +333,13 @@ pub fn parse_textual_date(lang: Language, textual_date: &str) -> Option Option { - parse_textual_date(lang, textual_date).map(OffsetDateTime::from) +pub fn parse_textual_date_to_dt( + lang: Language, + utc_offset: UtcOffset, + textual_date: &str, +) -> Option { + parse_textual_date(lang, textual_date) + .map(|parsed| OffsetDateTime::from(parsed).replace_offset(utc_offset)) } /// Parse a textual date (e.g. "29 minutes ago" "Jul 2, 2014") into a Date object. @@ -345,15 +350,17 @@ pub fn parse_textual_date_to_d( textual_date: &str, warnings: &mut Vec, ) -> Option { - parse_textual_date_or_warn(lang, textual_date, warnings).map(OffsetDateTime::date) + parse_textual_date_or_warn(lang, UtcOffset::UTC, textual_date, warnings) + .map(OffsetDateTime::date) } pub fn parse_textual_date_or_warn( lang: Language, + utc_offset: UtcOffset, textual_date: &str, warnings: &mut Vec, ) -> Option { - let res = parse_textual_date_to_dt(lang, textual_date); + let res = parse_textual_date_to_dt(lang, utc_offset, textual_date); if res.is_none() { warnings.push(format!("could not parse textual date `{textual_date}`")); } @@ -1101,11 +1108,13 @@ mod tests { #[test] fn t_to_datetime() { // Absolute date - let date = parse_textual_date_to_dt(Language::En, "Last updated on Jan 3, 2020").unwrap(); + let date = + parse_textual_date_to_dt(Language::En, UtcOffset::UTC, "Last updated on Jan 3, 2020") + .unwrap(); assert_eq!(date, datetime!(2020-1-3 0:00 +0)); // Relative date - let date = parse_textual_date_to_dt(Language::En, "1 year ago").unwrap(); + let date = parse_textual_date_to_dt(Language::En, UtcOffset::UTC, "1 year ago").unwrap(); let now = OffsetDateTime::now_utc(); assert_eq!(date.year(), now.year() - 1); }