fix: update large number samples

codegen/src/collect_album_types.rs

@@ -3,13 +3,13 @@ use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
 use futures::stream::{self, StreamExt};
 use path_macro::path;
 use rustypipe::{
-    client::{ClientType, RustyPipe, RustyPipeQuery, YTContext},
+    client::{ClientType, RustyPipe, RustyPipeQuery},
     model::AlbumType,
     param::{locale::LANGUAGES, Language},
 };
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 
-use crate::util::{self, TextRuns};
+use crate::util::{self, QBrowse, TextRuns};
 
 pub async fn collect_album_types(project_root: &Path, concurrency: usize) {
     let json_path = path!(project_root / "testfiles" / "dict" / "album_type_samples.json");
@@ -91,13 +91,6 @@ struct HeaderRenderer {
     subtitle: TextRuns,
 }
 
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct QBrowse<'a> {
-    context: YTContext<'a>,
-    browse_id: &'a str,
-}
-
 async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String {
     let context = query
         .get_context(ClientType::DesktopMusic, true, None)
@@ -105,6 +98,7 @@ async fn get_album_type(query: &RustyPipeQuery, id: &str) -> String {
     let body = QBrowse {
         context,
         browse_id: id,
+        params: None,
     };
     let response_txt = query
         .raw(ClientType::DesktopMusic, "browse", &body)

codegen/src/collect_large_numbers.rs

@@ -1,4 +1,5 @@
 use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
 use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
 
 use anyhow::{Context, Result};
@@ -6,20 +7,33 @@ use futures::{stream, StreamExt};
 use once_cell::sync::Lazy;
 use path_macro::path;
 use regex::Regex;
-use reqwest::{header, Client};
+use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery};
 use rustypipe::param::{locale::LANGUAGES, Language};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
-use serde_with::serde_as;
+use serde_with::{serde_as, DefaultOnError, VecSkipError};
-use serde_with::VecSkipError;
 
-use crate::util::{self, Text};
+use crate::util::{self, QBrowse, QCont, Text};
 
-type CollectedNumbers = BTreeMap<Language, BTreeMap<u8, (String, u64)>>;
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(untagged)]
+enum NumKey {
+    Mag(u8),
+    S(NumKeyS),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(rename_all = "lowercase")]
+enum NumKeyS {
+    Zero,
+    One,
+}
+
+type CollectedNumbers = BTreeMap<Language, BTreeMap<NumKey, (String, u64)>>;
 
 /// Collect video view count texts in every supported language
 /// and write them to `testfiles/dict/large_number_samples.json`.
 ///
-/// YouTube's API outputs the subscriber count of a channel only in a
+/// YouTube's API outputs subscriber and view counts only in a
 /// approximated format (e.g *880K subscribers*), which varies
 /// by language.
 ///
@@ -34,36 +48,61 @@ pub async fn collect_large_numbers(project_root: &Path, concurrency: usize) {
     let json_path = path!(project_root / "testfiles" / "dict" / "large_number_samples.json");
     let json_path_all =
         path!(project_root / "testfiles" / "dict" / "large_number_samples_all.json");
+    let rp = RustyPipe::new();
 
     let channels = [
-        "UCq-Fj5jknLsUf-MWSy4_brA", // 10e8 (225M)
+        "UCq-Fj5jknLsUf-MWSy4_brA", // 10e8 (241M)
-        "UCcdwLMPsaU2ezNSJU1nFoBQ", // 10e7 (60M)
+        "UCcdwLMPsaU2ezNSJU1nFoBQ", // 10e7 (67M)
-        "UC6mIxFTvXkWQVEHPsEdflzQ", // 10e6 (1.7M)
+        "UC6mIxFTvXkWQVEHPsEdflzQ", // 10e6 (1.8M)
-        "UCD0y51PJfvkZNe3y3FR5riw", // 10e5 (125K)
+        "UCD0y51PJfvkZNe3y3FR5riw", // 10e5 (126K)
-        "UCNcN0dW43zE0Om3278fjY8A", // 10e4 (27K)
+        "UCNcN0dW43zE0Om3278fjY8A", // 10e4 (33K)
         "UC0QEucPrn0-Ddi3JBTcs5Kw", // 10e3 (5K)
-        "UCXvtcj9xUQhaqPaitFf2DqA", // (170)
+        "UCXvtcj9xUQhaqPaitFf2DqA", // (275)
-        "UCq-XMc01T641v-4P3hQYJWg", // (636)
+        "UCq-XMc01T641v-4P3hQYJWg", // (695)
+        "UCaZL4eLD7a30Fa8QI-sRi_g", // (31K)
+        "UCO-dylEoJozPTxGYd8fTQxA", // (5)
+        "UCQXYK94vDqOEkPbTCyL0OjA", // (1)
     ];
 
-    let collected_numbers_all: BTreeMap<Language, BTreeMap<String, u64>> = stream::iter(LANGUAGES)
+    // Build a lookup table for the channel's subscriber counts
-        .map(|lang| async move {
+    let subscriber_counts: Arc<BTreeMap<String, u64>> = stream::iter(channels)
-            let mut entry = BTreeMap::new();
+        .map(|c| {
+            let rp = rp.query();
+            async move {
+                let channel = get_channel(&rp, c).await.unwrap();
 
-            for (n, ch_id) in channels.iter().enumerate() {
+                let n = util::parse_largenum_en(&channel.subscriber_count).unwrap();
-                let channel = get_channel(ch_id, lang)
+                (c.to_owned(), n)
-                    .await
-                    .context(format!("{lang}-{n}"))
-                    .unwrap();
-
-                channel.view_counts.iter().for_each(|(num, txt)| {
-                    entry.insert(txt.to_owned(), *num);
-                });
-
-                println!("collected {lang}-{n}");
             }
+        })
+        .buffer_unordered(concurrency)
+        .collect::<BTreeMap<_, _>>()
+        .await
+        .into();
 
-            (lang, entry)
+    let collected_numbers_all: BTreeMap<Language, BTreeMap<String, u64>> = stream::iter(LANGUAGES)
+        .map(|lang| {
+            let rp = rp.query().lang(lang);
+            let subscriber_counts = subscriber_counts.clone();
+            async move {
+                let mut entry = BTreeMap::new();
+
+                for (n, ch_id) in channels.iter().enumerate() {
+                    let channel = get_channel(&rp, ch_id)
+                        .await
+                        .context(format!("{lang}-{n}"))
+                        .unwrap();
+
+                    channel.view_counts.iter().for_each(|(num, txt)| {
+                        entry.insert(txt.to_owned(), *num);
+                    });
+                    entry.insert(channel.subscriber_count, subscriber_counts[*ch_id]);
+
+                    println!("collected {lang}-{n}");
+                }
+
+                (lang, entry)
+            }
         })
         .buffer_unordered(concurrency)
         .collect()
@@ -74,7 +113,15 @@ pub async fn collect_large_numbers(project_root: &Path, concurrency: usize) {
         .map(|(lang, entry)| {
             let mut e2 = BTreeMap::new();
             entry.iter().for_each(|(txt, num)| {
-                e2.insert(get_mag(*num), (txt.to_owned(), *num));
+                let key = if num == &0 {
+                    NumKey::S(NumKeyS::Zero)
+                } else if num == &1 {
+                    NumKey::S(NumKeyS::One)
+                } else {
+                    NumKey::Mag(get_mag(*num))
+                };
+
+                e2.insert(key, (txt.to_owned(), *num));
             });
             (*lang, e2)
         })
@@ -136,17 +183,22 @@ pub fn write_samples_to_dict(project_root: &Path) {
             .get(&lang)
             .unwrap()
             .iter()
-            .find_map(|(mag, (txt, _))| {
+            .find_map(|(key, (txt, _))| {
-                let point = POINT_REGEX
+                match key {
-                    .captures(txt)
+                    NumKey::Mag(mag) => {
-                    .map(|c| c.get(1).unwrap().as_str());
+                        let point = POINT_REGEX
+                            .captures(txt)
+                            .map(|c| c.get(1).unwrap().as_str());
 
-                if let Some(point) = point {
+                        if let Some(point) = point {
-                    let num_all = util::parse_numeric::<u64>(txt).unwrap();
+                            let num_all = util::parse_numeric::<u64>(txt).unwrap();
-                    // If the number parsed from all digits has the same order of
+                            // If the number parsed from all digits has the same order of
-                    // magnitude as the actual number, it must be a separator.
+                            // magnitude as the actual number, it must be a separator.
-                    // Otherwise it is a decimal point
+                            // Otherwise it is a decimal point
-                    return Some((get_mag(num_all) == *mag) ^ (point == ","));
+                            return Some((get_mag(num_all) == *mag) ^ (point == ","));
+                        }
+                    }
+                    NumKey::S(_) => {}
                 }
                 None
             })
@@ -182,42 +234,48 @@ pub fn write_samples_to_dict(project_root: &Path) {
         for lang in e_langs {
             let entry = collected_nums.get(&lang).unwrap();
 
-            entry.iter().for_each(|(mag, (txt, _))| {
+            entry.iter().for_each(|(key, (txt, _))| {
-                let filtered = util::filter_largenumstr(txt);
+                match key {
+                    NumKey::Mag(mag) => {
+                        let filtered = util::filter_largenumstr(txt);
 
-                let tokens: Vec<String> = match dict_entry.by_char {
+                        let tokens: Vec<String> = match dict_entry.by_char {
-                    true => filtered.chars().map(|c| c.to_string()).collect(),
+                            true => filtered.chars().map(|c| c.to_string()).collect(),
-                    false => filtered.split_whitespace().map(|c| c.to_string()).collect(),
+                            false => filtered.split_whitespace().map(|c| c.to_string()).collect(),
-                };
+                        };
 
-                let num_before_point =
+                        let num_before_point =
-                    util::parse_numeric::<u64>(txt.split(decimal_point).next().unwrap()).unwrap();
+                            util::parse_numeric::<u64>(txt.split(decimal_point).next().unwrap())
-                let mag_before_point = get_mag(num_before_point);
+                                .unwrap();
-                let mut mag_remaining = mag - mag_before_point;
+                        let mag_before_point = get_mag(num_before_point);
+                        let mut mag_remaining = mag - mag_before_point;
 
-                tokens.iter().for_each(|t| {
+                        tokens.iter().for_each(|t| {
-                    // These tokens are correct in all languages
+                            // These tokens are correct in all languages
-                    // and are used to parse combined prefixes like `1.1K crore` (en-IN)
+                            // and are used to parse combined prefixes like `1.1K crore` (en-IN)
-                    let known_tmag: u8 = if t.len() == 1 {
+                            let known_tmag: u8 = if t.len() == 1 {
-                        match t.as_str() {
+                                match t.as_str() {
-                            "K" | "k" => 3,
+                                    "K" | "k" => 3,
-                            // 'm' means 10^3 in Catalan, 'B' means 10^3 in Turkish
+                                    // 'm' means 10^3 in Catalan, 'B' means 10^3 in Turkish
-                            // 'M' means 10^9 in Indonesian
+                                    // 'M' means 10^9 in Indonesian
-                            _ => 0,
+                                    _ => 0,
-                        }
+                                }
-                    } else {
+                            } else {
-                        0
+                                0
-                    };
+                            };
 
-                    // K/M/B
+                            // K/M/B
-                    if known_tmag > 0 {
+                            if known_tmag > 0 {
-                        mag_remaining = mag_remaining
+                                mag_remaining = mag_remaining
-                            .checked_sub(known_tmag)
+                                    .checked_sub(known_tmag)
-                            .expect("known magnitude incorrect");
+                                    .expect("known magnitude incorrect");
-                    } else {
+                            } else {
-                        insert_token(t.to_owned(), mag_remaining);
+                                insert_token(t.to_owned(), mag_remaining);
+                            }
+                        });
                     }
-                });
+                    NumKey::S(_) => {}
+                }
             });
         }
 
@@ -250,6 +308,19 @@ YouTube channel videos response
 #[serde(rename_all = "camelCase")]
 struct Channel {
     contents: Contents,
+    header: ChannelHeader,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ChannelHeader {
+    c4_tabbed_header_renderer: HeaderRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct HeaderRenderer {
+    subscriber_count_text: Text,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -275,113 +346,212 @@ struct TabRendererWrap {
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct TabRenderer {
-    content: SectionListRendererWrap,
+    content: RichGridRendererWrap,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct SectionListRendererWrap {
+struct RichGridRendererWrap {
-    section_list_renderer: SectionListRenderer,
+    rich_grid_renderer: RichGridRenderer,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct SectionListRenderer {
-    contents: Vec<ItemSectionRendererWrap>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ItemSectionRendererWrap {
-    item_section_renderer: ItemSectionRenderer,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct ItemSectionRenderer {
-    contents: Vec<GridRendererWrap>,
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GridRendererWrap {
-    grid_renderer: GridRenderer,
 }
 
 #[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct GridRenderer {
+struct RichGridRenderer {
     #[serde_as(as = "VecSkipError<_>")]
-    items: Vec<VideoListItem>,
+    contents: Vec<RichItemRendererWrap>,
+    #[serde(default)]
+    #[serde_as(as = "DefaultOnError")]
+    header: Option<RichGridHeader>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct VideoListItem {
+struct RichItemRendererWrap {
-    grid_video_renderer: GridVideoRenderer,
+    rich_item_renderer: RichItemRenderer,
 }
 
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct GridVideoRenderer {
+struct RichItemRenderer {
+    content: VideoRendererWrap,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct VideoRendererWrap {
+    video_renderer: VideoRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct VideoRenderer {
     /// `24,194 views`
     view_count_text: Text,
     /// `19K views`
     short_view_count_text: Text,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RichGridHeader {
+    feed_filter_chip_bar_renderer: ChipBar,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ChipBar {
+    contents: Vec<Chip>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Chip {
+    chip_cloud_chip_renderer: ChipRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ChipRenderer {
+    navigation_endpoint: NavigationEndpoint,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct NavigationEndpoint {
+    continuation_command: ContinuationCommand,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ContinuationCommand {
+    token: String,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ContinuationResponse {
+    // #[serde_as(as = "VecSkipError<_>")]
+    on_response_received_actions: Vec<ContinuationAction>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ContinuationAction {
+    reload_continuation_items_command: ContinuationItemsWrap,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ContinuationItemsWrap {
+    #[serde_as(as = "VecSkipError<_>")]
+    continuation_items: Vec<RichItemRendererWrap>,
+}
+
 #[derive(Clone, Debug)]
 struct ChannelData {
-    view_counts: Vec<(u64, String)>,
+    view_counts: BTreeMap<u64, String>,
+    subscriber_count: String,
 }
 
-async fn get_channel(channel_id: &str, lang: Language) -> Result<ChannelData> {
+async fn get_channel(query: &RustyPipeQuery, channel_id: &str) -> Result<ChannelData> {
-    let client = Client::new();
+    let resp = query
+        .raw(
+            ClientType::DesktopMusic,
+            "browse",
+            &QBrowse {
+                context: query.get_context(ClientType::Desktop, true, None).await,
+                browse_id: channel_id,
+                params: Some("EgZ2aWRlb3MYASAAMAE"),
+            },
+        )
+        .await?;
 
-    let body = format!(
+    let channel = serde_json::from_str::<Channel>(&resp)?;
-        "{}{}{}{}{}",
-        r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":""##,
-        lang,
-        r##"","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}},"params":"EgZ2aWRlb3MYASAAMAE%3D","browseId":""##,
-        channel_id,
-        "\"}"
-    );
 
-    let resp = client
+    let tab = &channel.contents.two_column_browse_results_renderer.tabs[0]
-        .post("https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false")
+        .tab_renderer
-        .header(header::CONTENT_TYPE, "application/json")
+        .content
-        .body(body)
+        .rich_grid_renderer;
-        .send().await?
-        .error_for_status()?;
 
-    let channel = resp.json::<Channel>().await?;
+    let popular_token = tab.header.as_ref().and_then(|h| {
+        h.feed_filter_chip_bar_renderer.contents.get(1).map(|c| {
+            c.chip_cloud_chip_renderer
+                .navigation_endpoint
+                .continuation_command
+                .token
+                .to_owned()
+        })
+    });
+
+    let mut view_counts: BTreeMap<u64, String> = tab
+        .contents
+        .iter()
+        .map(|itm| {
+            let v = &itm.rich_item_renderer.content.video_renderer;
+            (
+                util::parse_numeric(&v.view_count_text.text).unwrap_or_default(),
+                v.short_view_count_text.text.to_owned(),
+            )
+        })
+        .collect();
+
+    if let Some(popular_token) = popular_token {
+        let resp = query
+            .raw(
+                ClientType::Desktop,
+                "browse",
+                &QCont {
+                    context: query.get_context(ClientType::Desktop, true, None).await,
+                    continuation: &popular_token,
+                },
+            )
+            .await?;
+
+        let continuation = serde_json::from_str::<ContinuationResponse>(&resp)?;
+
+        continuation
+            .on_response_received_actions
+            .iter()
+            .for_each(|a| {
+                a.reload_continuation_items_command
+                    .continuation_items
+                    .iter()
+                    .for_each(|itm| {
+                        let v = &itm.rich_item_renderer.content.video_renderer;
+                        view_counts.insert(
+                            util::parse_numeric(&v.view_count_text.text).unwrap(),
+                            v.short_view_count_text.text.to_owned(),
+                        );
+                    })
+            });
+    }
 
     Ok(ChannelData {
-        view_counts: channel
+        view_counts,
-            .contents
+        subscriber_count: channel
-            .two_column_browse_results_renderer
+            .header
-            .tabs
+            .c4_tabbed_header_renderer
-            .get(0)
+            .subscriber_count_text
-            .map(|tab| {
+            .text,
-                tab.tab_renderer.content.section_list_renderer.contents[0]
-                    .item_section_renderer
-                    .contents[0]
-                    .grid_renderer
-                    .items
-                    .iter()
-                    .map(|itm| {
-                        (
-                            util::parse_numeric(&itm.grid_video_renderer.view_count_text.text)
-                                .unwrap(),
-                            itm.grid_video_renderer
-                                .short_view_count_text
-                                .text
-                                .to_owned(),
-                        )
-                    })
-                    .collect()
-            })
-            .unwrap_or_default(),
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use rustypipe::client::RustyPipe;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn t() {
+        let rp = RustyPipe::new();
+        let x = get_channel(&rp.query(), "UCQXYK94vDqOEkPbTCyL0OjA")
+            .await
+            .unwrap();
+        dbg!(&x);
+    }
+}

codegen/src/util.rs

@@ -8,7 +8,7 @@ use std::{
 
 use once_cell::sync::Lazy;
 use path_macro::path;
-use rustypipe::{model::AlbumType, param::Language};
+use rustypipe::{client::YTContext, model::AlbumType, param::Language};
 use serde::{Deserialize, Serialize};
 
 static DICT_PATH: Lazy<PathBuf> = Lazy::new(|| path!("testfiles" / "dict" / "dictionary.json"));
@@ -58,6 +58,22 @@ pub struct DictEntry {
     pub album_types: BTreeMap<String, AlbumType>,
 }
 
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct QBrowse<'a> {
+    pub context: YTContext<'a>,
+    pub browse_id: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub params: Option<&'a str>,
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct QCont<'a> {
+    pub context: YTContext<'a>,
+    pub continuation: &'a str,
+}
+
 #[derive(Clone, Debug, Deserialize)]
 pub struct TextRuns {
     pub runs: Vec<Text>,
@@ -100,7 +116,19 @@ pub fn filter_datestr(string: &str) -> String {
 pub fn filter_largenumstr(string: &str) -> String {
     string
         .chars()
-        .filter(|c| !matches!(c, '\u{200b}' | '.' | ',') && !c.is_ascii_digit())
+        .filter(|c| {
+            !matches!(
+                c,
+                '\u{200b}'
+                    | '\u{202b}'
+                    | '\u{202c}'
+                    | '\u{202e}'
+                    | '\u{200e}'
+                    | '\u{200f}'
+                    | '.'
+                    | ','
+            ) && !c.is_ascii_digit()
+        })
         .collect()
 }
 
@@ -140,3 +168,40 @@ where
 
     numbers
 }
+
+pub fn parse_largenum_en(string: &str) -> Option<u64> {
+    let (num, mut exp, filtered) = {
+        let mut buf = String::new();
+        let mut filtered = String::new();
+        let mut exp = 0;
+        let mut after_point = false;
+        for c in string.chars() {
+            if c.is_ascii_digit() {
+                buf.push(c);
+
+                if after_point {
+                    exp -= 1;
+                }
+            } else if c == '.' {
+                after_point = true;
+            } else if !matches!(c, '\u{200b}' | '.' | ',') {
+                filtered.push(c);
+            }
+        }
+        (buf.parse::<u64>().ok()?, exp, filtered)
+    };
+
+    let lookup_token = |token: &str| match token {
+        "K" => Some(3),
+        "M" => Some(6),
+        "B" => Some(9),
+        _ => None,
+    };
+
+    exp += filtered
+        .split_whitespace()
+        .filter_map(lookup_token)
+        .sum::<i32>();
+
+    num.checked_mul((10_u64).checked_pow(exp.try_into().ok()?)?)
+}

src/util/dictionary.rs

@@ -339,22 +339,24 @@ pub(crate) fn entry(lang: Language) -> Entry {
             },
             comma_decimal: false,
             number_tokens: ::phf::Map {
-                key: 10121458955350035957,
+                key: 12913932095322966823,
                 disps: &[
-                    (0, 5),
+                    (0, 7),
-                    (6, 0),
+                    (9, 8),
+                    (0, 0),
                 ],
                 entries: &[
-                    ("ল\u{9be}", 5),
-                    ("কোঃট\u{9be}", 9),
-                    ("নিঃট\u{9be}", 6),
-                    ("হ\u{9be}জ\u{9be}ৰ", 3),
-                    ("ল\u{9be}খট\u{9be}", 5),
-                    ("নিয\u{9c1}ত", 6),
                     ("হ\u{9be}", 3),
-                    ("ল\u{9be}খ", 5),
                     ("হ\u{9be}জ\u{9be}ৰট\u{9be}", 3),
                     ("নিয\u{9c1}তট\u{9be}", 6),
+                    ("হ\u{9be}জ\u{9be}ৰ", 3),
+                    ("ল\u{9be}", 5),
+                    ("ল\u{9be}খট\u{9be}", 5),
+                    ("কোঃট\u{9be}", 9),
+                    ("নিঃট\u{9be}", 6),
+                    ("নিয\u{9c1}ত", 6),
+                    ("নিঃ", 6),
+                    ("ল\u{9be}খ", 5),
                 ],
             },
             album_types: ::phf::Map {
@@ -851,14 +853,14 @@ pub(crate) fn entry(lang: Language) -> Entry {
             },
             comma_decimal: true,
             number_tokens: ::phf::Map {
-                key: 12913932095322966823,
+                key: 7485420634051515786,
                 disps: &[
                     (2, 0),
                 ],
                 entries: &[
-                    ("mM", 9),
-                    ("M", 6),
                     ("m", 3),
+                    ("kM", 9),
+                    ("M", 6),
                 ],
             },
             album_types: ::phf::Map {
@@ -3181,18 +3183,14 @@ pub(crate) fn entry(lang: Language) -> Entry {
             },
             comma_decimal: false,
             number_tokens: ::phf::Map {
-                key: 2980949210194914378,
+                key: 12913932095322966823,
                 disps: &[
-                    (1, 3),
+                    (1, 0),
-                    (5, 0),
                 ],
                 entries: &[
-                    ("억명", 8),
+                    ("천", 3),
-                    ("천명", 3),
+                    ("만", 4),
-                    ("만회", 4),
+                    ("억", 8),
-                    ("천회", 3),
-                    ("억회", 8),
-                    ("만명", 4),
                 ],
             },
             album_types: ::phf::Map {
@@ -3964,18 +3962,18 @@ pub(crate) fn entry(lang: Language) -> Entry {
             },
             comma_decimal: false,
             number_tokens: ::phf::Map {
-                key: 10121458955350035957,
+                key: 12913932095322966823,
                 disps: &[
-                    (5, 1),
+                    (3, 0),
-                    (2, 0),
+                    (0, 2),
                 ],
                 entries: &[
-                    ("ထောင\u{103a}", 3),
+                    ("ထ", 3),
-                    ("သန\u{103a}း", 6),
-                    ("က\u{102f}ဋေထ", 10),
                     ("က\u{102f}ဋေ", 7),
-                    ("သောင\u{103a}း", 4),
+                    ("သန\u{103a}း", 6),
+                    ("ထောင\u{103a}", 3),
                     ("သ\u{102d}န\u{103a}း", 5),
+                    ("သောင\u{103a}း", 4),
                 ],
             },
             album_types: ::phf::Map {

src/util/mod.rs

@@ -335,7 +335,7 @@ where
         _ => dict_entry.number_tokens.get(token).map(|t| *t as i32),
     };
 
-    if dict_entry.by_char {
+    if dict_entry.by_char || lang == Language::Ko {
         exp += filtered
             .chars()
             .filter_map(|token| lookup_token(&token.to_string()))
@@ -511,7 +511,7 @@ pub(crate) mod tests {
     fn t_parse_large_numstr_samples() {
         let json_path = path!(*TESTFILES / "dict" / "large_number_samples.json");
         let json_file = File::open(json_path).unwrap();
-        let number_samples: BTreeMap<Language, BTreeMap<u8, (String, u64)>> =
+        let number_samples: BTreeMap<Language, BTreeMap<String, (String, u64)>> =
             serde_json::from_reader(BufReader::new(json_file)).unwrap();
 
         number_samples.iter().for_each(|(lang, entry)| {
@@ -540,12 +540,17 @@ pub(crate) mod tests {
         // in the string.
         let rounded = {
             let n_significant_d = string.chars().filter(char::is_ascii_digit).count();
-            let mag = (expect as f64).log10().floor();
+            if n_significant_d == 0 {
-            let factor = 10_u64.pow(1 + mag as u32 - n_significant_d as u32);
+                expect
-            (((expect as f64) / factor as f64).floor() as u64) * factor
+            } else {
+                let mag = (expect as f64).log10().floor();
+                let factor = 10_u64.pow(1 + mag as u32 - n_significant_d as u32);
+                (((expect as f64) / factor as f64).floor() as u64) * factor
+            }
         };
 
-        let res = parse_large_numstr::<u64>(string, lang).expect(string);
+        // TODO: add support for zero values
+        let res = parse_large_numstr::<u64>(string, lang).unwrap_or_default();
         assert_eq!(res, rounded, "{string} (lang: {lang}, exact: {expect})");
     }
 }

testfiles/dict/dictionary.json

@@ -180,7 +180,8 @@
       "লাখটা": 5,
       "হা": 3,
       "হাজাৰ": 3,
-      "হাজাৰটা": 3
+      "হাজাৰটা": 3,
+      "নিঃ": 6
     },
     "album_types": {
       "ep": "Ep",
@@ -486,7 +487,7 @@
     "number_tokens": {
       "M": 6,
       "m": 3,
-      "mM": 9
+      "kM": 9
     },
     "album_types": {
       "audiollibre": "Audiobook",
@@ -1870,12 +1871,9 @@
     },
     "comma_decimal": false,
     "number_tokens": {
-      "만명": 4,
+      "만": 4,
-      "만회": 4,
+      "억": 8,
-      "억명": 8,
+      "천": 3
-      "억회": 8,
-      "천명": 3,
-      "천회": 3
     },
     "album_types": {
       "ep": "Ep",
@@ -2332,11 +2330,11 @@
     "comma_decimal": false,
     "number_tokens": {
       "ကုဋေ": 7,
-      "ကုဋေထ": 10,
       "ထောင်": 3,
       "သန်း": 6,
       "သိန်း": 5,
-      "သောင်း": 4
+      "သောင်း": 4,
+      "ထ": 3
     },
     "album_types": {
       "ep": "Ep",