feat: add number_tokens for parsing large nums to dictionary

codegen/src/collect_large_numbers.rs

@@ -0,0 +1,358 @@
+use std::collections::HashMap;
+use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};
+
+use anyhow::{Context, Result};
+use fancy_regex::Regex;
+use futures::{stream, StreamExt};
+use once_cell::sync::Lazy;
+use reqwest::{header, Client};
+use rustypipe::model::{locale::LANGUAGES, Language};
+use serde::Deserialize;
+use serde_with::serde_as;
+use serde_with::VecSkipError;
+
+use crate::util::{self, Text};
+
+type CollectedNumbers = BTreeMap<Language, BTreeMap<u8, (String, u64)>>;
+
+/// Collect video view count texts in every supported language
+/// and write them to `testfiles/dict/large_number_samples.json`.
+///
+/// YouTube's API outputs the subscriber count of a channel only in a
+/// approximated format (e.g *880K subscribers*), which varies
+/// by language.
+///
+/// To parse these numbers correctly we need to collect textual numbers
+/// of different orders of magnitude in every language. This script extracts
+/// the view count texts from the most popular videos of different channels.
+///
+/// We extract these instead of subscriber counts because the YouTube API
+/// outputs view counts both in approximated and exact format, so we can use
+/// the exact counts to figure out the tokens.
+pub async fn collect_large_numbers(project_root: &Path, concurrency: usize) {
+    let mut json_path = project_root.to_path_buf();
+    json_path.push("testfiles/dict/large_number_samples.json");
+
+    let channels = [
+        "UCq-Fj5jknLsUf-MWSy4_brA", // 10e8 (225M)
+        "UCcdwLMPsaU2ezNSJU1nFoBQ", // 10e7 (60M)
+        "UC6mIxFTvXkWQVEHPsEdflzQ", // 10e6 (1.7M)
+        "UCD0y51PJfvkZNe3y3FR5riw", // 10e5 (125K)
+        "UCNcN0dW43zE0Om3278fjY8A", // 10e4 (27K)
+        "UC0QEucPrn0-Ddi3JBTcs5Kw", // 10e3 (5K)
+        "UCGiJh0NZ52wRhYKYnuZI08Q", // 10e1 (37)
+    ];
+
+    let collected_numbers: CollectedNumbers = stream::iter(LANGUAGES)
+        .map(|lang| async move {
+            let mut entry = BTreeMap::new();
+
+            for (n, ch_id) in channels.iter().enumerate() {
+                let channel = get_channel(ch_id, lang)
+                    .await
+                    .context(format!("{}-{}", lang, n))
+                    .unwrap();
+
+                channel.view_counts.iter().for_each(|(num, txt)| {
+                    entry.insert(get_mag(*num), (txt.to_owned(), *num));
+                });
+
+                println!("collected {}-{}", lang, n);
+            }
+
+            (lang, entry)
+        })
+        .buffer_unordered(concurrency)
+        .collect()
+        .await;
+
+    let file = File::create(json_path).unwrap();
+    serde_json::to_writer_pretty(file, &collected_numbers).unwrap();
+}
+
+/// Attempt to parse the numbers collected by `collect-large-numbers`
+/// and write the results to `dictionary.json`.
+pub fn write_samples_to_dict(project_root: &Path) {
+    let mut json_path = project_root.to_path_buf();
+    json_path.push("testfiles/dict/large_number_samples.json");
+
+    let json_file = File::open(json_path).unwrap();
+    let collected_nums: CollectedNumbers =
+        serde_json::from_reader(BufReader::new(json_file)).unwrap();
+    let mut dict = util::read_dict(project_root);
+    let langs = dict.keys().map(|k| k.to_owned()).collect::<Vec<_>>();
+
+    static POINT_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"\d(\.|,)\d{1,3}(?:\D|$)").unwrap());
+
+    for lang in langs {
+        let dict_entry = dict.entry(lang).or_default();
+
+        let mut e_langs = dict_entry.equivalent.clone();
+        e_langs.push(lang);
+
+        let comma_decimal = collected_nums
+            .get(&lang)
+            .unwrap()
+            .iter()
+            .find_map(|(mag, (txt, _))| {
+                let point = POINT_REGEX
+                    .captures(txt)
+                    .unwrap()
+                    .map(|c| c.get(1).unwrap().as_str());
+
+                if let Some(point) = point {
+                    let num_all = util::parse_numeric::<u64>(txt).unwrap();
+                    // If the number parsed from all digits has the same order of
+                    // magnitude as the actual number, it must be a separator.
+                    // Otherwise it is a decimal point
+                    return Some((get_mag(num_all) == *mag) ^ (point == ","));
+                }
+                None
+            })
+            .unwrap();
+
+        let decimal_point = match comma_decimal {
+            true => ",",
+            false => ".",
+        };
+
+        // Search for tokens
+
+        // This map holds all the tokens we encounter while parsing the language
+        // If a new token is found, it is stored in this map with the derived order of
+        // magnitude.
+        // If the token is found again with a different derived order of magnitude,
+        // its value in the map is set to None.
+        let mut found_tokens: HashMap<String, Option<u8>> = HashMap::new();
+
+        let mut insert_token = |token: String, mag: u8| {
+            let found_token = found_tokens.entry(token).or_insert(match mag {
+                0 => None,
+                x => Some(x),
+            });
+
+            if let Some(f) = found_token {
+                if *f != mag {
+                    *found_token = None;
+                }
+            }
+        };
+
+        for lang in e_langs {
+            let entry = collected_nums.get(&lang).unwrap();
+
+            entry.iter().for_each(|(mag, (txt, _))| {
+                let filtered = util::filter_largenumstr(txt);
+
+                let tokens: Vec<String> = match dict_entry.by_char {
+                    true => filtered.chars().map(|c| c.to_string()).collect(),
+                    false => filtered.split_whitespace().map(|c| c.to_string()).collect(),
+                };
+
+                let num_before_point =
+                    util::parse_numeric::<u64>(txt.split(decimal_point).next().unwrap()).unwrap();
+                let mag_before_point = get_mag(num_before_point);
+                let mut mag_remaining = mag - mag_before_point;
+
+                tokens.iter().for_each(|t| {
+                    // These tokens are correct in all languages
+                    // and are used to parse combined prefixes like `1.1K crore` (en-IN)
+                    let known_tmag: u8 = if t.len() == 1 {
+                        match t.as_str() {
+                            "K" | "k" => 3,
+                            "M" => 6,
+                            // 'm' means 10^3 in Catalan, 'B' means 10^3 in Turkish
+                            _ => 0,
+                        }
+                    } else {
+                        0
+                    };
+
+                    // K/M/B
+                    if known_tmag > 0 {
+                        mag_remaining = mag_remaining
+                            .checked_sub(known_tmag)
+                            .expect("known magnitude incorrect");
+                    } else {
+                        insert_token(t.to_owned(), mag_remaining);
+                    }
+                });
+            });
+        }
+
+        // Insert collected data into dictionary
+        dict_entry.number_tokens = found_tokens
+            .into_iter()
+            .filter_map(|(k, v)| v.map(|v| (k, v)))
+            .collect();
+        dict_entry.comma_decimal = comma_decimal;
+    }
+
+    util::write_dict(project_root, &dict);
+}
+
+fn get_mag(n: u64) -> u8 {
+    (n as f64).log10().floor() as u8
+}
+
+/*
+YouTube channel videos response
+*/
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Channel {
+    contents: Contents,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Contents {
+    two_column_browse_results_renderer: TabsRenderer,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TabsRenderer {
+    #[serde_as(as = "VecSkipError<_>")]
+    tabs: Vec<TabRendererWrap>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TabRendererWrap {
+    tab_renderer: TabRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TabRenderer {
+    content: SectionListRendererWrap,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SectionListRendererWrap {
+    section_list_renderer: SectionListRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SectionListRenderer {
+    contents: Vec<ItemSectionRendererWrap>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ItemSectionRendererWrap {
+    item_section_renderer: ItemSectionRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ItemSectionRenderer {
+    contents: Vec<GridRendererWrap>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GridRendererWrap {
+    grid_renderer: GridRenderer,
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GridRenderer {
+    #[serde_as(as = "VecSkipError<_>")]
+    items: Vec<VideoListItem>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct VideoListItem {
+    grid_video_renderer: GridVideoRenderer,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct GridVideoRenderer {
+    /// `24,194 views`
+    view_count_text: Text,
+    /// `19K views`
+    short_view_count_text: Text,
+}
+
+#[derive(Clone, Debug)]
+struct ChannelData {
+    view_counts: Vec<(u64, String)>,
+}
+
+async fn get_channel(channel_id: &str, lang: Language) -> Result<ChannelData> {
+    let client = Client::new();
+
+    let body = format!(
+        "{}{}{}{}{}",
+        r##"{"context":{"client":{"clientName":"WEB","clientVersion":"2.20220914.06.00","platform":"DESKTOP","originalUrl":"https://www.youtube.com/","hl":""##,
+        lang,
+        r##"","gl":"US"},"request":{"internalExperimentFlags":[],"useSsl":true},"user":{"lockedSafetyMode":false}},"params":"EgZ2aWRlb3MYASAAMAE%3D","browseId":""##,
+        channel_id,
+        "\"}"
+    );
+
+    let resp = client
+        .post("https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false")
+        .header(header::CONTENT_TYPE, "application/json")
+        .body(body)
+        .send().await?
+        .error_for_status()?;
+
+    let channel = resp.json::<Channel>().await?;
+
+    Ok(ChannelData {
+        view_counts: channel
+            .contents
+            .two_column_browse_results_renderer
+            .tabs
+            .get(0)
+            .map(|tab| {
+                tab.tab_renderer.content.section_list_renderer.contents[0]
+                    .item_section_renderer
+                    .contents[0]
+                    .grid_renderer
+                    .items
+                    .iter()
+                    .map(|itm| {
+                        (
+                            util::parse_numeric(
+                                &itm.grid_video_renderer.view_count_text.simple_text,
+                            )
+                            .unwrap(),
+                            itm.grid_video_renderer
+                                .short_view_count_text
+                                .simple_text
+                                .to_owned(),
+                        )
+                    })
+                    .collect()
+            })
+            .unwrap_or_default(),
+    })
+}
+
+#[tokio::test]
+async fn test() {
+    let channel = get_channel("UCcdwLMPsaU2ezNSJU1nFoBQ", Language::Az)
+        .await
+        .unwrap();
+
+    dbg!(channel);
+}
+
+#[test]
+fn test2() {
+    write_samples_to_dict(Path::new(
+        "/home/thetadev/Documents/Programmieren/Rust/rustypipe",
+    ));
+}