From 0659a64361d30c8623f30c0f7d7349ac386e8da2 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 23 Sep 2022 19:42:49 +0200 Subject: [PATCH] feat: implemented large number parser --- src/client/channel.rs | 12 +++++- src/client/video_details.rs | 86 +++++++++++++++++++++++-------------- src/model/mod.rs | 9 ++-- src/util.rs | 16 +++++-- 4 files changed, 79 insertions(+), 44 deletions(-) diff --git a/src/client/channel.rs b/src/client/channel.rs index f7adfa7..d7f82e9 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -2,7 +2,11 @@ use anyhow::{bail, Result}; use reqwest::Method; use serde::Serialize; -use crate::{model::ChannelVideos, serializer::MapResult}; +use crate::{ + model::{ChannelVideos, Paginator}, + serializer::MapResult, + util, +}; use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; @@ -53,7 +57,7 @@ impl MapResponse for response::Channel { fn map_response( self, id: &str, - _lang: crate::model::Language, + lang: crate::model::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result> { let warnings = Vec::new(); @@ -72,6 +76,10 @@ impl MapResponse for response::Channel { c: ChannelVideos { id: header.channel_id, name: header.title, + subscriber_count: header + .subscriber_count_text + .and_then(|txt| util::parse_large_numstr(&txt, lang)), + videos: Paginator::default(), }, warnings, }) diff --git a/src/client/video_details.rs b/src/client/video_details.rs index d179df8..e931f75 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -177,14 +177,12 @@ impl MapResponse for response::VideoDetails { _ => bail!("could not find primary_info"), }; - /* - TODO: use large number parser for this let comment_count = comment_count_section.and_then(|s| { - util::parse_numeric_or_warn::( + util::parse_large_numstr::( &s.comments_entry_point_header_renderer.comment_count, - &mut warnings, + lang, ) - });*/ + }); let comment_ctoken = comment_ctoken_section.map(|s| { s.continuation_item_renderer @@ -301,8 +299,9 @@ impl MapResponse for response::VideoDetails { name: channel_name, avatar: owner.thumbnail.into(), verification: owner.badges.into(), - subscriber_count: None, - subscriber_count_txt: owner.subscriber_count_text, + subscriber_count: owner + .subscriber_count_text + .and_then(|txt| util::parse_large_numstr(&txt, lang)), }, view_count, like_count, @@ -312,8 +311,8 @@ impl MapResponse for response::VideoDetails { is_ccommons, chapters, recommended, - top_comments: Paginator::new(None, Vec::new(), comment_ctoken), - latest_comments: Paginator::new(None, Vec::new(), latest_comments_ctoken), + top_comments: Paginator::new(comment_count, Vec::new(), comment_ctoken), + latest_comments: Paginator::new(comment_count, Vec::new(), latest_comments_ctoken), }, warnings, }) @@ -431,7 +430,6 @@ fn map_recommendations( avatar: video.channel_thumbnail.into(), verification: video.owner_badges.into(), subscriber_count: None, - subscriber_count_txt: None, }, publish_date: video.published_time_text.as_ref().and_then(|txt| { timeago::parse_timeago_or_warn(lang, txt, &mut warnings) @@ -514,7 +512,6 @@ fn map_comment( .map(|b| b.author_comment_badge_renderer.icon.into()) .unwrap_or_default(), subscriber_count: None, - subscriber_count_txt: None, }), _ => None, }, @@ -620,7 +617,11 @@ mod tests { assert_eq!(details.channel.name, "SMTOWN"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::Verified); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 30000000, "expected >30M subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 30000000, + "expected >30M subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 232000000, @@ -644,11 +645,11 @@ mod tests { assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); - // assert!( - // details.top_comments.count.unwrap() > 700000, - // "expected > 700K comments, got {}", - // details.top_comments.count.unwrap() - // ); + assert!( + details.top_comments.count.unwrap() > 700000, + "expected > 700K comments, got {}", + details.top_comments.count.unwrap() + ); assert!(!details.top_comments.is_exhausted()); assert!(!details.latest_comments.is_exhausted()); } @@ -671,7 +672,11 @@ mod tests { assert_eq!(details.channel.name, "Sentamusic"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::Artist); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 33000, "expected >33K subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 33000, + "expected >33K subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 20309, @@ -730,7 +735,11 @@ mod tests { assert_eq!(details.channel.name, "media.ccc.de"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::None); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 170000, "expected >170K subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 170000, + "expected >170K subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 2517358, @@ -754,11 +763,11 @@ mod tests { assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); - // assert!( - // details.top_comments.count.unwrap() > 700000, - // "expected > 700K comments, got {}", - // details.top_comments.count.unwrap() - // ); + assert!( + details.top_comments.count.unwrap() > 2199, + "expected > 2199 comments, got {}", + details.top_comments.count.unwrap() + ); assert!(!details.top_comments.is_exhausted()); assert!(!details.latest_comments.is_exhausted()); } @@ -961,7 +970,11 @@ mod tests { assert_eq!(details.channel.name, "Linus Tech Tips"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::Verified); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 14700000, "expected >14.7M subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 14700000, + "expected >14.7M subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 1157262, @@ -1036,11 +1049,11 @@ mod tests { assert!(!details.recommended.items.is_empty()); assert!(!details.recommended.is_exhausted()); - // assert!( - // details.top_comments.count.unwrap() > 700000, - // "expected > 700K comments, got {}", - // details.top_comments.count.unwrap() - // ); + assert!( + details.top_comments.count.unwrap() > 3199, + "expected > 3199 comments, got {}", + details.top_comments.count.unwrap() + ); assert!(!details.top_comments.is_exhausted()); assert!(!details.latest_comments.is_exhausted()); } @@ -1057,7 +1070,6 @@ mod tests { details.title, "🌎 Nasa Live Stream - Earth From Space : Live Views from the ISS" ); - // TODO: not full description insta::assert_yaml_snapshot!(details.description, @r###" --- - Text: "Live NASA - Views Of Earth from Space\nLive video feed of Earth from the International Space Station (ISS) Cameras\n-----------------------------------------------------------------------------------------------------\nWatch our latest video - The Sun - 4K Video / Solar Flares\n" @@ -1088,7 +1100,11 @@ mod tests { assert_eq!(details.channel.name, "Space Videos"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::Verified); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 5500000, "expected >5.5M subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 5500000, + "expected >5.5M subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 10, @@ -1140,7 +1156,11 @@ mod tests { assert_eq!(details.channel.name, "PrinceOfFALLEN"); assert!(!details.channel.avatar.is_empty(), "no channel avatars"); assert_eq!(details.channel.verification, Verification::None); - // TODO: assert!(details.channel.subscriber_count.unwrap() > 1400, "expected >1400 subs, got {}", details.channel.subscriber_count); + assert!( + details.channel.subscriber_count.unwrap() > 1400, + "expected >1400 subs, got {}", + details.channel.subscriber_count.unwrap() + ); assert!( details.view_count > 200, diff --git a/src/model/mod.rs b/src/model/mod.rs index 63c4704..a95f13f 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -329,10 +329,10 @@ pub struct Channel { pub verification: Verification, /// Approximate number of subscribers /// + /// `None` if hidden by the owner or not present. + /// /// Info: This is only present in the `VideoDetails` response - pub subscriber_count: Option, - /// Textual subscriber count (e.g `1.41M subscribers`, depends on language) - pub subscriber_count_txt: Option, + pub subscriber_count: Option, } /* @@ -400,14 +400,13 @@ pub struct ChannelVideos { pub id: String, /// Channel name pub name: String, - /* /// Channel subscriber count /// /// `None` if the subscriber count was hidden by the owner /// or could not be parsed. pub subscriber_count: Option, + /// Videos fetched from the channel pub videos: Paginator, - */ } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] diff --git a/src/util.rs b/src/util.rs index b80379d..bd3e542 100644 --- a/src/util.rs +++ b/src/util.rs @@ -230,7 +230,11 @@ impl TryRemove for Vec { } } -fn parse_large_numstr(string: &str, lang: Language) -> Option { +/// Parse a large, textual number (e.g. `1.4M subscribers`, `22K views`) +pub fn parse_large_numstr(string: &str, lang: Language) -> Option +where + F: TryFrom, +{ let dict_entry = dictionary::entry(lang); let decimal_point = match dict_entry.comma_decimal { true => ',', @@ -275,10 +279,14 @@ fn parse_large_numstr(string: &str, lang: Language) -> Option { .sum::(); } - num.checked_mul(some_or_bail!( - (10_u64).checked_pow(ok_or_bail!(exp.try_into(), None)), + F::try_from(some_or_bail!( + num.checked_mul(some_or_bail!( + (10_u64).checked_pow(ok_or_bail!(exp.try_into(), None)), + None + )), None )) + .ok() } #[cfg(test)] @@ -407,7 +415,7 @@ mod tests { (((expect as f64) / factor as f64).floor() as u64) * factor }; - let res = parse_large_numstr(string, lang).expect(string); + let res = parse_large_numstr::(string, lang).expect(string); assert_eq!( res, rounded, "{} (lang: {}, exact: {})",