From 6bb0c3792e4213a1a3127364615623b7e5d6d8ba Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 8 Sep 2022 23:01:31 +0200 Subject: [PATCH] add date to playlist --- Cargo.toml | 5 +- src/client/player.rs | 23 ++- src/client/playlist.rs | 21 ++- ...layer__tests__map_player_data_android.snap | 2 +- ...layer__tests__map_player_data_desktop.snap | 2 +- ...__tests__map_player_data_desktopmusic.snap | 2 +- ...t__player__tests__map_player_data_ios.snap | 2 +- ...__tests__map_player_data_tvhtml5embed.snap | 2 +- ...aylist__tests__map_playlist_data_long.snap | 2 +- ...ist__tests__map_playlist_data_nomusic.snap | 2 +- ...ylist__tests__map_playlist_data_short.snap | 2 +- src/codegen/collect_playlist_dates.rs | 2 +- src/model/mod.rs | 6 +- src/timeago.rs | 156 ++++++++++-------- 14 files changed, 129 insertions(+), 100 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f531af1..7ad4d82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" async-trait = "0.1.56" chrono = {version = "0.4.19", features = ["serde"]} +chronoutil = "0.2.3" futures = "0.3.21" indicatif = "0.17.0" filenamify = "0.1.0" @@ -36,8 +37,6 @@ env_logger = "0.9.0" test-log = "0.2.11" rstest = "0.15.0" temp_testdir = "0.2.3" -insta = "1.17.1" +insta = {version = "1.17.1", features = ["redactions"]} velcro = "0.5.3" -unic-langid = "0.9.0" -intl_pluralrules = "7.0.1" phf_codegen = "0.11.1" diff --git a/src/client/player.rs b/src/client/player.rs index bdb5e3a..2044caa 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -5,7 +5,7 @@ use std::{ }; use anyhow::{anyhow, bail, Result}; -use chrono::{NaiveDateTime, NaiveTime, TimeZone, Utc}; +use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone}; use fancy_regex::Regex; use log::{error, warn}; use once_cell::sync::Lazy; @@ -367,7 +367,7 @@ fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result insta::dynamic_redaction(move |value, _path| { + if is_desktop { + assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00")); + "2019-05-30T00:00:00" + } else { + assert_eq!(value, insta::internals::Content::None); + "~" + } + }), + }); } /// Assert equality within 10% margin @@ -572,10 +584,7 @@ mod tests { assert_eq!(player_data.info.is_live_content, false); if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic { - assert_eq!( - player_data.info.publish_date.unwrap().to_string(), - "2013-05-05 00:00:00 UTC" - ); + assert!(player_data.info.publish_date.unwrap().to_string().starts_with("2013-05-05 00:00:00")); assert_eq!(player_data.info.category.unwrap(), "Music"); assert_eq!(player_data.info.is_family_safe.unwrap(), true); } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 5806f9b..5b979a7 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -3,9 +3,9 @@ use reqwest::Method; use serde::Serialize; use crate::{ - model::{Channel, Playlist, Thumbnail, Video}, + model::{Channel, Language, Playlist, Thumbnail, Video}, serializer::text::{PageType, TextLink}, - util, + timeago, util, }; use super::{response, ClientType, ContextYT, RustyTube}; @@ -46,7 +46,7 @@ impl RustyTube { let playlist_response = serde_json::from_str::(&resp_body).context(resp_body)?; - map_playlist(&playlist_response) + map_playlist(&playlist_response, self.localization.language) } pub async fn get_playlist_cont(&self, playlist: &mut Playlist) -> Result<()> { @@ -95,7 +95,7 @@ impl RustyTube { } } -fn map_playlist(response: &response::Playlist) -> Result { +fn map_playlist(response: &response::Playlist, lang: Language) -> Result { let video_items = &some_or_bail!( some_or_bail!( some_or_bail!( @@ -228,7 +228,10 @@ fn map_playlist(response: &response::Playlist) -> Result { thumbnails, description, channel, - last_update: None, + last_update: match &last_update_txt { + Some(textual_date) => timeago::parse_textual_date_to_dt(lang, textual_date), + None => None, + }, last_update_txt, }) } @@ -383,15 +386,17 @@ mod tests { #[case::long("long")] #[case::short("short")] #[case::nomusic("nomusic")] - fn t_map_player_data(#[case] name: &str) { + fn t_map_playlist_data(#[case] name: &str) { let filename = format!("testfiles/playlist/playlist_{}.json", name); let json_path = Path::new(&filename); let json_file = File::open(json_path).unwrap(); let playlist: response::Playlist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let playlist_data = map_playlist(&playlist).unwrap(); - insta::assert_yaml_snapshot!(format!("map_playlist_data_{}", name), playlist_data); + let playlist_data = map_playlist(&playlist, Language::En).unwrap(); + insta::assert_yaml_snapshot!(format!("map_playlist_data_{}", name), playlist_data, { + ".last_update" => "[date]" + }); } #[test_log::test(tokio::test)] diff --git a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap index 4e9c4b2..845ccda 100644 --- a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap +++ b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap @@ -23,7 +23,7 @@ info: channel: id: UCbxxEi-ImPlbLx5F-fHetEg name: RomanSenykMusic - Royalty Free Music - publish_date: ~ + publish_date: "~" view_count: 426567 keywords: - no copyright music diff --git a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap index 532fd4b..a403140 100644 --- a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap +++ b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap @@ -26,7 +26,7 @@ info: channel: id: UCbxxEi-ImPlbLx5F-fHetEg name: RomanSenykMusic - Royalty Free Music - publish_date: "2019-05-30T00:00:00Z" + publish_date: "2019-05-30T00:00:00" view_count: 426567 keywords: - no copyright music diff --git a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap index d6cb5e2..1ad62f4 100644 --- a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap +++ b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap @@ -20,7 +20,7 @@ info: channel: id: UCbxxEi-ImPlbLx5F-fHetEg name: Romansenykmusic - publish_date: "2019-05-30T00:00:00Z" + publish_date: "2019-05-30T00:00:00" view_count: 426583 keywords: - no copyright music diff --git a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap index b7e2eee..a36c970 100644 --- a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap +++ b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap @@ -20,7 +20,7 @@ info: channel: id: UCbxxEi-ImPlbLx5F-fHetEg name: RomanSenykMusic - Royalty Free Music - publish_date: ~ + publish_date: "~" view_count: 426567 keywords: - no copyright music diff --git a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap index c15f215..3dee60b 100644 --- a/src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap +++ b/src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap @@ -26,7 +26,7 @@ info: channel: id: UCbxxEi-ImPlbLx5F-fHetEg name: RomanSenykMusic - Royalty Free Music - publish_date: ~ + publish_date: "~" view_count: 426567 keywords: - no copyright music diff --git a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_long.snap b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_long.snap index 515ca0c..e91bb3c 100644 --- a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_long.snap +++ b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_long.snap @@ -1924,6 +1924,6 @@ description: ~ channel: id: UCIekuFeMaV78xYfvpmoCnPg name: Best Music -last_update: ~ +last_update: "[date]" last_update_txt: "Last updated on Aug 7, 2022" diff --git a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_nomusic.snap b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_nomusic.snap index 813ce8d..fc9232c 100644 --- a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_nomusic.snap +++ b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_nomusic.snap @@ -1278,6 +1278,6 @@ description: "SHINE - Survival Hardcore in New Environment: Auf einem Server mac channel: id: UCQM0bS4_04-Y4JuYrgmnpZQ name: Chaosflo44 -last_update: ~ +last_update: "[date]" last_update_txt: "Last updated on Jul 2, 2014" diff --git a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_short.snap b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_short.snap index af701ee..3686b45 100644 --- a/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_short.snap +++ b/src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_short.snap @@ -1862,6 +1862,6 @@ thumbnails: height: 1200 description: ~ channel: ~ -last_update: ~ +last_update: "[date]" last_update_txt: Updated today diff --git a/src/codegen/collect_playlist_dates.rs b/src/codegen/collect_playlist_dates.rs index fcf1f4b..fd071c9 100644 --- a/src/codegen/collect_playlist_dates.rs +++ b/src/codegen/collect_playlist_dates.rs @@ -183,7 +183,7 @@ fn write_samples_to_dict() { // n days ago { let datestr = datestr_table.get(&DateCase::Ago).unwrap(); - let tago = timeago::parse(lang, &datestr); + let tago = timeago::parse_timeago(lang, &datestr); assert_eq!( tago, Some(TimeAgo { diff --git a/src/model/mod.rs b/src/model/mod.rs index bd76b1b..7853896 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -6,7 +6,7 @@ pub use locale::{Country, Language}; use std::ops::Range; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Local}; use serde::{Deserialize, Serialize}; pub trait FileFormat { @@ -33,7 +33,7 @@ pub struct Playlist { pub thumbnails: Vec, pub description: Option, pub channel: Option, - pub last_update: Option>, + pub last_update: Option>, pub last_update_txt: Option, } @@ -45,7 +45,7 @@ pub struct VideoInfo { pub length: u32, pub thumbnails: Vec, pub channel: Channel, - pub publish_date: Option>, + pub publish_date: Option>, pub view_count: u64, pub keywords: Vec, pub category: Option, diff --git a/src/timeago.rs b/src/timeago.rs index 71ed3d7..5b47ab7 100644 --- a/src/timeago.rs +++ b/src/timeago.rs @@ -1,11 +1,11 @@ -use std::{cmp::Ordering, ops::Mul}; +use std::ops::Mul; -use chrono::NaiveDate; +use chrono::{DateTime, Duration, Local, NaiveDate, NaiveDateTime, NaiveTime, TimeZone}; use serde::{Deserialize, Serialize}; use crate::{dictionary, model::Language, util}; -#[derive(Debug, Copy, Clone, Serialize, Deserialize, Eq)] +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct TimeAgo { pub n: u8, pub unit: TimeUnit, @@ -17,13 +17,13 @@ pub struct TaToken { pub unit: Option, } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum ParsedDate { +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ParsedDate { Absolute(NaiveDate), Relative(TimeAgo), } -#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "lowercase")] pub enum TimeUnit { Second, @@ -41,44 +41,6 @@ pub enum DateCmp { D, } -impl TimeUnit { - fn seconds(&self) -> u64 { - match self { - TimeUnit::Second => 1, - TimeUnit::Minute => 60, - TimeUnit::Hour => 3600, - TimeUnit::Day => 24 * 3600, - TimeUnit::Week => 7 * 24 * 3600, - TimeUnit::Month => 30 * 24 * 3600, - TimeUnit::Year => 365 * 24 * 3600, - } - } -} - -impl TimeAgo { - fn seconds(&self) -> u64 { - self.n as u64 * self.unit.seconds() - } -} - -impl PartialEq for TimeAgo { - fn eq(&self, other: &Self) -> bool { - self.seconds() == other.seconds() - } -} - -impl Ord for TimeAgo { - fn cmp(&self, other: &Self) -> Ordering { - self.seconds().cmp(&other.seconds()) - } -} - -impl PartialOrd for TimeAgo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - impl Mul for TimeAgo { type Output = Self; @@ -90,6 +52,32 @@ impl Mul for TimeAgo { } } +impl Into> for TimeAgo { + fn into(self) -> DateTime { + let ts = Local::now(); + match self.unit { + TimeUnit::Second => ts - Duration::seconds(self.n as i64), + TimeUnit::Minute => ts - Duration::minutes(self.n as i64), + TimeUnit::Hour => ts - Duration::hours(self.n as i64), + TimeUnit::Day => ts - Duration::days(self.n as i64), + TimeUnit::Week => ts - Duration::weeks(self.n as i64), + TimeUnit::Month => chronoutil::shift_months(ts, -(self.n as i32)), + TimeUnit::Year => chronoutil::shift_years(ts, -(self.n as i32)), + } + } +} + +impl Into> for ParsedDate { + fn into(self) -> DateTime { + match self { + ParsedDate::Absolute(date) => Local + .from_local_datetime(&NaiveDateTime::new(date, NaiveTime::from_hms(0, 0, 0))) + .unwrap(), + ParsedDate::Relative(timeago) => timeago.into(), + } + } +} + pub fn filter_str(string: &str) -> String { string .to_lowercase() @@ -153,7 +141,7 @@ fn parse_textual_month(entry: &dictionary::Entry, filtered_str: &str) -> Option< } } -pub fn parse(lang: Language, textual_date: &str) -> Option { +pub fn parse_timeago(lang: Language, textual_date: &str) -> Option { let entry = dictionary::entry(lang); let filtered_str = filter_str(textual_date); @@ -162,7 +150,11 @@ pub fn parse(lang: Language, textual_date: &str) -> Option { parse_ta_token(&entry, false, &filtered_str).map(|ta| ta * qu) } -fn parse_date(lang: Language, textual_date: &str) -> Option { +pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option> { + parse_timeago(lang, textual_date).map(|ta| ta.into()) +} + +pub fn parse_textual_date(lang: Language, textual_date: &str) -> Option { let entry = dictionary::entry(lang); let filtered_str = filter_str(textual_date); @@ -195,11 +187,10 @@ fn parse_date(lang: Language, textual_date: &str) -> Option { } match (y, m, d) { - (Some(y), Some(m), Some(d)) => Some(ParsedDate::Absolute(NaiveDate::from_ymd( - y.into(), - m.into(), - d.into(), - ))), + (Some(y), Some(m), Some(d)) => { + NaiveDate::from_ymd_opt(y.into(), m.into(), d.into()) + .map(|d| ParsedDate::Absolute(d)) + } _ => None, } } else { @@ -210,10 +201,15 @@ fn parse_date(lang: Language, textual_date: &str) -> Option { } } +pub fn parse_textual_date_to_dt(lang: Language, textual_date: &str) -> Option> { + parse_textual_date(lang, textual_date).map(|ta| ta.into()) +} + #[cfg(test)] mod tests { use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path}; + use chrono::Datelike; use rstest::rstest; use super::*; @@ -228,7 +224,7 @@ mod tests { #[case] textual_date: &str, #[case] expect: Option, ) { - let time_ago = parse(lang, textual_date); + let time_ago = parse_timeago(lang, textual_date); assert_eq!(time_ago, expect); } @@ -395,7 +391,7 @@ mod tests { assert_eq!(strings.len(), expect.len()); strings.iter().enumerate().for_each(|(n, s)| { assert_eq!( - parse(*lang, s), + parse_timeago(*lang, s), Some(expect[n]), "Language: {}, n: {}", lang, @@ -426,7 +422,7 @@ mod tests { timeago_table.entries.iter().for_each(|(lang, entries)| { entries.iter().for_each(|(t, entry)| { entry.cases.iter().for_each(|(txt, n)| { - let timeago = parse(*lang, txt); + let timeago = parse_timeago(*lang, txt); assert_eq!( timeago, Some(TimeAgo { n: *n, unit: *t }), @@ -458,7 +454,7 @@ mod tests { #[case] textual_date: &str, #[case] expect: Option, ) { - let parsed_date = parse_date(lang, textual_date); + let parsed_date = parse_textual_date(lang, textual_date); assert_eq!(parsed_date, expect); } @@ -471,7 +467,7 @@ mod tests { date_samples.iter().for_each(|(lang, samples)| { assert_eq!( - parse_date(*lang, samples.get("Today").unwrap()), + parse_textual_date(*lang, samples.get("Today").unwrap()), Some(ParsedDate::Relative(TimeAgo { n: 0, unit: TimeUnit::Day @@ -480,7 +476,7 @@ mod tests { lang ); assert_eq!( - parse_date(*lang, samples.get("Yesterday").unwrap()), + parse_textual_date(*lang, samples.get("Yesterday").unwrap()), Some(ParsedDate::Relative(TimeAgo { // YT's Singhalese translation has an error (yesterday == today) n: match lang { @@ -493,7 +489,7 @@ mod tests { lang ); assert_eq!( - parse_date(*lang, samples.get("Ago").unwrap()), + parse_textual_date(*lang, samples.get("Ago").unwrap()), Some(ParsedDate::Relative(TimeAgo { n: 3, unit: TimeUnit::Day @@ -502,77 +498,97 @@ mod tests { lang ); assert_eq!( - parse_date(*lang, samples.get("Jan").unwrap()), + parse_textual_date(*lang, samples.get("Jan").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2020, 1, 3))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Feb").unwrap()), + parse_textual_date(*lang, samples.get("Feb").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2016, 2, 7))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Mar").unwrap()), + parse_textual_date(*lang, samples.get("Mar").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2015, 3, 9))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Apr").unwrap()), + parse_textual_date(*lang, samples.get("Apr").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2017, 4, 2))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("May").unwrap()), + parse_textual_date(*lang, samples.get("May").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 5, 22))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Jun").unwrap()), + parse_textual_date(*lang, samples.get("Jun").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 6, 28))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Jul").unwrap()), + parse_textual_date(*lang, samples.get("Jul").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 7, 2))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Aug").unwrap()), + parse_textual_date(*lang, samples.get("Aug").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2015, 8, 23))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Sep").unwrap()), + parse_textual_date(*lang, samples.get("Sep").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2018, 9, 16))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Oct").unwrap()), + parse_textual_date(*lang, samples.get("Oct").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 10, 31))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Nov").unwrap()), + parse_textual_date(*lang, samples.get("Nov").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2016, 11, 3))), "lang: {}", lang ); assert_eq!( - parse_date(*lang, samples.get("Dec").unwrap()), + parse_textual_date(*lang, samples.get("Dec").unwrap()), Some(ParsedDate::Absolute(NaiveDate::from_ymd(2021, 12, 24))), "lang: {}", lang ); }) } + + #[test] + fn t_to_datetime() { + // Absolute date + let date = parse_textual_date_to_dt(Language::En, "Last updated on Jan 3, 2020").unwrap(); + assert_eq!( + date, + Local + .from_local_datetime(&NaiveDateTime::new( + NaiveDate::from_ymd(2020, 1, 3), + NaiveTime::from_hms(0, 0, 0) + )) + .unwrap() + ); + + // Relative date + let date = parse_textual_date_to_dt(Language::En, "1 year ago").unwrap(); + let now = Local::now(); + assert_eq!(date.year(), now.year() - 1); + } }