Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add date to playlist

Browse source
This commit is contained in:
ThetaDev 2022-09-08 23:01:31 +02:00
parent 9ddf9a3ac4
commit 6bb0c3792e
14 changed files with 129 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Cargo.toml
View file

@ -25,6 +25,7 @@ serde_with = {version = "2.0.0", features = ["json"] }
rand = "0.8.5"
async-trait = "0.1.56"
chrono = {version = "0.4.19", features = ["serde"]}
chronoutil = "0.2.3"
futures = "0.3.21"
indicatif = "0.17.0"
filenamify = "0.1.0"
@ -36,8 +37,6 @@ env_logger = "0.9.0"
test-log = "0.2.11"
rstest = "0.15.0"
temp_testdir = "0.2.3"
insta = "1.17.1"
insta = {version = "1.17.1", features = ["redactions"]}
velcro = "0.5.3"
unic-langid = "0.9.0"
intl_pluralrules = "7.0.1"
phf_codegen = "0.11.1"

23
src/client/player.rs
View file

@ -5,7 +5,7 @@ use std::{
};
use anyhow::{anyhow, bail, Result};
use chrono::{NaiveDateTime, NaiveTime, TimeZone, Utc};
use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
use fancy_regex::Regex;
use log::{error, warn};
use once_cell::sync::Lazy;
@ -367,7 +367,7 @@ fn map_player_data(response: response::Player, deobf: &Deobfuscator) -> Result<V
},
publish_date: microformat.as_ref().map(|m| {
let ndt = NaiveDateTime::new(m.publish_date, NaiveTime::from_hms(0, 0, 0));
Utc.from_local_datetime(&ndt).unwrap()
Local.from_local_datetime(&ndt).unwrap()
}),
view_count: video_details.view_count,
keywords: video_details
@ -526,7 +526,19 @@ mod tests {
let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let player_data = map_player_data(resp, &DEOBFUSCATOR).unwrap();
insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), player_data)
let is_desktop = name == "desktop" || name == "desktopmusic";
insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), player_data, {
".info.publish_date" => insta::dynamic_redaction(move |value, _path| {
if is_desktop {
assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00"));
"2019-05-30T00:00:00"
} else {
assert_eq!(value, insta::internals::Content::None);
"~"
}
}),
});
}
/// Assert equality within 10% margin
@ -572,10 +584,7 @@ mod tests {
assert_eq!(player_data.info.is_live_content, false);
if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic {
assert_eq!(
player_data.info.publish_date.unwrap().to_string(),
"2013-05-05 00:00:00 UTC"
);
assert!(player_data.info.publish_date.unwrap().to_string().starts_with("2013-05-05 00:00:00"));
assert_eq!(player_data.info.category.unwrap(), "Music");
assert_eq!(player_data.info.is_family_safe.unwrap(), true);
}

21
src/client/playlist.rs
View file

@ -3,9 +3,9 @@ use reqwest::Method;
use serde::Serialize;
use crate::{
model::{Channel, Playlist, Thumbnail, Video},
model::{Channel, Language, Playlist, Thumbnail, Video},
serializer::text::{PageType, TextLink},
util,
timeago, util,
};
use super::{response, ClientType, ContextYT, RustyTube};
@ -46,7 +46,7 @@ impl RustyTube {
let playlist_response =
serde_json::from_str::<response::Playlist>(&resp_body).context(resp_body)?;
map_playlist(&playlist_response)
map_playlist(&playlist_response, self.localization.language)
}
pub async fn get_playlist_cont(&self, playlist: &mut Playlist) -> Result<()> {
@ -95,7 +95,7 @@ impl RustyTube {
}
}
fn map_playlist(response: &response::Playlist) -> Result<Playlist> {
fn map_playlist(response: &response::Playlist, lang: Language) -> Result<Playlist> {
let video_items = &some_or_bail!(
some_or_bail!(
some_or_bail!(
@ -228,7 +228,10 @@ fn map_playlist(response: &response::Playlist) -> Result<Playlist> {
thumbnails,
description,
channel,
last_update: None,
last_update: match &last_update_txt {
Some(textual_date) => timeago::parse_textual_date_to_dt(lang, textual_date),
None => None,
},
last_update_txt,
})
}
@ -383,15 +386,17 @@ mod tests {
#[case::long("long")]
#[case::short("short")]
#[case::nomusic("nomusic")]
fn t_map_player_data(#[case] name: &str) {
fn t_map_playlist_data(#[case] name: &str) {
let filename = format!("testfiles/playlist/playlist_{}.json", name);
let json_path = Path::new(&filename);
let json_file = File::open(json_path).unwrap();
let playlist: response::Playlist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let playlist_data = map_playlist(&playlist).unwrap();
insta::assert_yaml_snapshot!(format!("map_playlist_data_{}", name), playlist_data);
let playlist_data = map_playlist(&playlist, Language::En).unwrap();
insta::assert_yaml_snapshot!(format!("map_playlist_data_{}", name), playlist_data, {
".last_update" => "[date]"
});
}
#[test_log::test(tokio::test)]

2
src/client/snapshots/rustypipe__client__player__tests__map_player_data_android.snap
View file

@ -23,7 +23,7 @@ info:
channel:
id: UCbxxEi-ImPlbLx5F-fHetEg
name: RomanSenykMusic - Royalty Free Music
publish_date: ~
publish_date: "~"
view_count: 426567
keywords:
- no copyright music

2
src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktop.snap
View file

@ -26,7 +26,7 @@ info:
channel:
id: UCbxxEi-ImPlbLx5F-fHetEg
name: RomanSenykMusic - Royalty Free Music
publish_date: "2019-05-30T00:00:00Z"
publish_date: "2019-05-30T00:00:00"
view_count: 426567
keywords:
- no copyright music

2
src/client/snapshots/rustypipe__client__player__tests__map_player_data_desktopmusic.snap
View file

@ -20,7 +20,7 @@ info:
channel:
id: UCbxxEi-ImPlbLx5F-fHetEg
name: Romansenykmusic
publish_date: "2019-05-30T00:00:00Z"
publish_date: "2019-05-30T00:00:00"
view_count: 426583
keywords:
- no copyright music

2
src/client/snapshots/rustypipe__client__player__tests__map_player_data_ios.snap
View file

@ -20,7 +20,7 @@ info:
channel:
id: UCbxxEi-ImPlbLx5F-fHetEg
name: RomanSenykMusic - Royalty Free Music
publish_date: ~
publish_date: "~"
view_count: 426567
keywords:
- no copyright music

2
src/client/snapshots/rustypipe__client__player__tests__map_player_data_tvhtml5embed.snap
View file

@ -26,7 +26,7 @@ info:
channel:
id: UCbxxEi-ImPlbLx5F-fHetEg
name: RomanSenykMusic - Royalty Free Music
publish_date: ~
publish_date: "~"
view_count: 426567
keywords:
- no copyright music

2
src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_long.snap
View file

@ -1924,6 +1924,6 @@ description: ~
channel:
id: UCIekuFeMaV78xYfvpmoCnPg
name: Best Music
last_update: ~
last_update: "[date]"
last_update_txt: "Last updated on Aug 7, 2022"

2
src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_nomusic.snap
View file

@ -1278,6 +1278,6 @@ description: "SHINE - Survival Hardcore in New Environment: Auf einem Server mac
channel:
id: UCQM0bS4_04-Y4JuYrgmnpZQ
name: Chaosflo44
last_update: ~
last_update: "[date]"
last_update_txt: "Last updated on Jul 2, 2014"

2
src/client/snapshots/rustypipe__client__playlist__tests__map_playlist_data_short.snap
View file

@ -1862,6 +1862,6 @@ thumbnails:
height: 1200
description: ~
channel: ~
last_update: ~
last_update: "[date]"
last_update_txt: Updated today

2
src/codegen/collect_playlist_dates.rs
View file

@ -183,7 +183,7 @@ fn write_samples_to_dict() {
// n days ago
{
let datestr = datestr_table.get(&DateCase::Ago).unwrap();
let tago = timeago::parse(lang, &datestr);
let tago = timeago::parse_timeago(lang, &datestr);
assert_eq!(
tago,
Some(TimeAgo {

6
src/model/mod.rs
View file

@ -6,7 +6,7 @@ pub use locale::{Country, Language};
use std::ops::Range;
use chrono::{DateTime, Utc};
use chrono::{DateTime, Local};
use serde::{Deserialize, Serialize};
pub trait FileFormat {
@ -33,7 +33,7 @@ pub struct Playlist {
pub thumbnails: Vec<Thumbnail>,
pub description: Option<String>,
pub channel: Option<Channel>,
pub last_update: Option<DateTime<Utc>>,
pub last_update: Option<DateTime<Local>>,
pub last_update_txt: Option<String>,
}
@ -45,7 +45,7 @@ pub struct VideoInfo {
pub length: u32,
pub thumbnails: Vec<Thumbnail>,
pub channel: Channel,
pub publish_date: Option<DateTime<Utc>>,
pub publish_date: Option<DateTime<Local>>,
pub view_count: u64,
pub keywords: Vec<String>,
pub category: Option<String>,

156
src/timeago.rs
View file

@ -1,11 +1,11 @@
use std::{cmp::Ordering, ops::Mul};
use std::ops::Mul;
use chrono::NaiveDate;
use chrono::{DateTime, Duration, Local, NaiveDate, NaiveDateTime, NaiveTime, TimeZone};
use serde::{Deserialize, Serialize};
use crate::{dictionary, model::Language, util};
#[derive(Debug, Copy, Clone, Serialize, Deserialize, Eq)]
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct TimeAgo {
pub n: u8,
pub unit: TimeUnit,
@ -17,13 +17,13 @@ pub struct TaToken {
pub unit: Option<TimeUnit>,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum ParsedDate {
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ParsedDate {
Absolute(NaiveDate),
Relative(TimeAgo),
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "lowercase")]
pub enum TimeUnit {
Second,
@ -41,44 +41,6 @@ pub enum DateCmp {
D,
}
impl TimeUnit {
fn seconds(&self) -> u64 {
match self {
TimeUnit::Second => 1,
TimeUnit::Minute => 60,
TimeUnit::Hour => 3600,
TimeUnit::Day => 24 * 3600,
TimeUnit::Week => 7 * 24 * 3600,
TimeUnit::Month => 30 * 24 * 3600,
TimeUnit::Year => 365 * 24 * 3600,
}
}
}
impl TimeAgo {
fn seconds(&self) -> u64 {
self.n as u64 * self.unit.seconds()
}
}
impl PartialEq for TimeAgo {
fn eq(&self, other: &Self) -> bool {
self.seconds() == other.seconds()
}
}
impl Ord for TimeAgo {
fn cmp(&self, other: &Self) -> Ordering {
self.seconds().cmp(&other.seconds())
}
}
impl PartialOrd for TimeAgo {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Mul<u8> for TimeAgo {
type Output = Self;
@ -90,6 +52,32 @@ impl Mul<u8> for TimeAgo {
}
}
impl Into<DateTime<Local>> for TimeAgo {
fn into(self) -> DateTime<Local> {
let ts = Local::now();
match self.unit {
TimeUnit::Second => ts - Duration::seconds(self.n as i64),
TimeUnit::Minute => ts - Duration::minutes(self.n as i64),
TimeUnit::Hour => ts - Duration::hours(self.n as i64),
TimeUnit::Day => ts - Duration::days(self.n as i64),
TimeUnit::Week => ts - Duration::weeks(self.n as i64),
TimeUnit::Month => chronoutil::shift_months(ts, -(self.n as i32)),
TimeUnit::Year => chronoutil::shift_years(ts, -(self.n as i32)),
}
}
}
impl Into<DateTime<Local>> for ParsedDate {
fn into(self) -> DateTime<Local> {
match self {
ParsedDate::Absolute(date) => Local
.from_local_datetime(&NaiveDateTime::new(date, NaiveTime::from_hms(0, 0, 0)))
.unwrap(),
ParsedDate::Relative(timeago) => timeago.into(),
}
}
}
pub fn filter_str(string: &str) -> String {
string
.to_lowercase()
@ -153,7 +141,7 @@ fn parse_textual_month(entry: &dictionary::Entry, filtered_str: &str) -> Option<
}
}
pub fn parse(lang: Language, textual_date: &str) -> Option<TimeAgo> {
pub fn parse_timeago(lang: Language, textual_date: &str) -> Option<TimeAgo> {
let entry = dictionary::entry(lang);
let filtered_str = filter_str(textual_date);
@ -162,7 +150,11 @@ pub fn parse(lang: Language, textual_date: &str) -> Option<TimeAgo> {
parse_ta_token(&entry, false, &filtered_str).map(|ta| ta * qu)
}
fn parse_date(lang: Language, textual_date: &str) -> Option<ParsedDate> {
pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option<DateTime<Local>> {
parse_timeago(lang, textual_date).map(|ta| ta.into())
}
pub fn parse_textual_date(lang: Language, textual_date: &str) -> Option<ParsedDate> {
let entry = dictionary::entry(lang);
let filtered_str = filter_str(textual_date);
@ -195,11 +187,10 @@ fn parse_date(lang: Language, textual_date: &str) -> Option<ParsedDate> {
}
match (y, m, d) {
(Some(y), Some(m), Some(d)) => Some(ParsedDate::Absolute(NaiveDate::from_ymd(
y.into(),
m.into(),
d.into(),
))),
(Some(y), Some(m), Some(d)) => {
NaiveDate::from_ymd_opt(y.into(), m.into(), d.into())
.map(|d| ParsedDate::Absolute(d))
}
_ => None,
}
} else {
@ -210,10 +201,15 @@ fn parse_date(lang: Language, textual_date: &str) -> Option<ParsedDate> {
}
}
pub fn parse_textual_date_to_dt(lang: Language, textual_date: &str) -> Option<DateTime<Local>> {
parse_textual_date(lang, textual_date).map(|ta| ta.into())
}
#[cfg(test)]
mod tests {
use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
use chrono::Datelike;
use rstest::rstest;
use super::*;
@ -228,7 +224,7 @@ mod tests {
#[case] textual_date: &str,
#[case] expect: Option<TimeAgo>,
) {
let time_ago = parse(lang, textual_date);
let time_ago = parse_timeago(lang, textual_date);
assert_eq!(time_ago, expect);
}
@ -395,7 +391,7 @@ mod tests {
assert_eq!(strings.len(), expect.len());
strings.iter().enumerate().for_each(|(n, s)| {
assert_eq!(
parse(*lang, s),
parse_timeago(*lang, s),
Some(expect[n]),
"Language: {}, n: {}",
lang,
@ -426,7 +422,7 @@ mod tests {
timeago_table.entries.iter().for_each(|(lang, entries)| {
entries.iter().for_each(|(t, entry)| {
entry.cases.iter().for_each(|(txt, n)| {
let timeago = parse(*lang, txt);
let timeago = parse_timeago(*lang, txt);
assert_eq!(
timeago,
Some(TimeAgo { n: *n, unit: *t }),
@ -458,7 +454,7 @@ mod tests {
#[case] textual_date: &str,
#[case] expect: Option<ParsedDate>,
) {
let parsed_date = parse_date(lang, textual_date);
let parsed_date = parse_textual_date(lang, textual_date);
assert_eq!(parsed_date, expect);
}
@ -471,7 +467,7 @@ mod tests {
date_samples.iter().for_each(|(lang, samples)| {
assert_eq!(
parse_date(*lang, samples.get("Today").unwrap()),
parse_textual_date(*lang, samples.get("Today").unwrap()),
Some(ParsedDate::Relative(TimeAgo {
n: 0,
unit: TimeUnit::Day
@ -480,7 +476,7 @@ mod tests {
lang
);
assert_eq!(
parse_date(*lang, samples.get("Yesterday").unwrap()),
parse_textual_date(*lang, samples.get("Yesterday").unwrap()),
Some(ParsedDate::Relative(TimeAgo {
// YT's Singhalese translation has an error (yesterday == today)
n: match lang {
@ -493,7 +489,7 @@ mod tests {
lang
);
assert_eq!(
parse_date(*lang, samples.get("Ago").unwrap()),
parse_textual_date(*lang, samples.get("Ago").unwrap()),
Some(ParsedDate::Relative(TimeAgo {
n: 3,
unit: TimeUnit::Day
@ -502,77 +498,97 @@ mod tests {
lang
);
assert_eq!(
parse_date(*lang, samples.get("Jan").unwrap()),
parse_textual_date(*lang, samples.get("Jan").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2020, 1, 3))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Feb").unwrap()),
parse_textual_date(*lang, samples.get("Feb").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2016, 2, 7))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Mar").unwrap()),
parse_textual_date(*lang, samples.get("Mar").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2015, 3, 9))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Apr").unwrap()),
parse_textual_date(*lang, samples.get("Apr").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2017, 4, 2))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("May").unwrap()),
parse_textual_date(*lang, samples.get("May").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 5, 22))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Jun").unwrap()),
parse_textual_date(*lang, samples.get("Jun").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 6, 28))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Jul").unwrap()),
parse_textual_date(*lang, samples.get("Jul").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 7, 2))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Aug").unwrap()),
parse_textual_date(*lang, samples.get("Aug").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2015, 8, 23))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Sep").unwrap()),
parse_textual_date(*lang, samples.get("Sep").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2018, 9, 16))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Oct").unwrap()),
parse_textual_date(*lang, samples.get("Oct").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2014, 10, 31))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Nov").unwrap()),
parse_textual_date(*lang, samples.get("Nov").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2016, 11, 3))),
"lang: {}",
lang
);
assert_eq!(
parse_date(*lang, samples.get("Dec").unwrap()),
parse_textual_date(*lang, samples.get("Dec").unwrap()),
Some(ParsedDate::Absolute(NaiveDate::from_ymd(2021, 12, 24))),
"lang: {}",
lang
);
})
}
#[test]
fn t_to_datetime() {
// Absolute date
let date = parse_textual_date_to_dt(Language::En, "Last updated on Jan 3, 2020").unwrap();
assert_eq!(
date,
Local
.from_local_datetime(&NaiveDateTime::new(
NaiveDate::from_ymd(2020, 1, 3),
NaiveTime::from_hms(0, 0, 0)
))
.unwrap()
);
// Relative date
let date = parse_textual_date_to_dt(Language::En, "1 year ago").unwrap();
let now = Local::now();
assert_eq!(date.year(), now.year() - 1);
}
}