Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add large number parser

Browse source
This commit is contained in:
ThetaDev 2022-09-23 18:19:24 +02:00
parent 5d19259a14
commit fc7655093b
5 changed files with 192 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/client/player.rs
View file

@ -298,20 +298,17 @@ fn deobf_nsig(
last_nsig: &mut [String; 2],
) -> Result<()> {
let nsig: String;
match url_params.get("n") {
Some(n) => {
nsig = if n == &last_nsig[0] {
last_nsig[1].to_owned()
} else {
let nsig = deobf.deobfuscate_nsig(n)?;
last_nsig[0] = n.to_string();
last_nsig[1] = nsig.to_owned();
nsig
};
if let Some(n) = url_params.get("n") {
nsig = if n == &last_nsig[0] {
last_nsig[1].to_owned()
} else {
let nsig = deobf.deobfuscate_nsig(n)?;
last_nsig[0] = n.to_string();
last_nsig[1] = nsig.to_owned();
nsig
};
url_params.insert("n".to_owned(), nsig);
}
None => {}
url_params.insert("n".to_owned(), nsig);
};
Ok(())
}

69
src/dictionary.rs
View file

@ -294,18 +294,16 @@ pub fn entry(lang: Language) -> Entry {
},
comma_decimal: false,
number_tokens: ::phf::Map {
key: 15467950696543387533,
key: 12913932095322966823,
disps: &[
(1, 0),
(4, 5),
(3, 0),
],
entries: &[
("নিয\u{9c1}তট\u{9be}", 6),
("নিঃট\u{9be}", 6),
("ল\u{9be}খট\u{9be}", 5),
("শঃ", 9),
("কোঃট\u{9be}", 9),
("হ\u{9be}জ\u{9be}ৰট\u{9be}", 3),
("নিঃট\u{9be}", 6),
("কোঃট\u{9be}", 9),
("ল\u{9be}খট\u{9be}", 5),
],
},
},
@ -568,12 +566,13 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 15467950696543387533,
disps: &[
(1, 0),
(0, 0),
],
entries: &[
("ল\u{9be}টি", 5),
("শত", 9),
("শত", 2),
("হ\u{9be}টি", 3),
("কোটি", 7),
],
},
},
@ -716,12 +715,13 @@ pub fn entry(lang: Language) -> Entry {
},
comma_decimal: true,
number_tokens: ::phf::Map {
key: 15467950696543387533,
key: 12913932095322966823,
disps: &[
(0, 0),
(2, 0),
],
entries: &[
("mM", 9),
("M", 6),
("m", 3),
],
},
@ -1044,14 +1044,15 @@ pub fn entry(lang: Language) -> Entry {
},
comma_decimal: false,
number_tokens: ::phf::Map {
key: 15467950696543387533,
key: 14108922650502679131,
disps: &[
(1, 0),
],
entries: &[
("crore", 7),
("B", 9),
("lakh", 5),
("crore", 7),
("M", 6),
("B", 9),
],
},
},
@ -1118,10 +1119,11 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
(1, 0),
],
entries: &[
("mil", 9),
("mil", 3),
("M", 6),
],
},
},
@ -1188,10 +1190,11 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
(1, 0),
],
entries: &[
("mil", 9),
("mil", 3),
("M", 6),
],
},
},
@ -1328,8 +1331,10 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
],
entries: &[
("M", 6),
],
},
},
@ -1512,6 +1517,7 @@ pub fn entry(lang: Language) -> Entry {
(0, 0),
],
entries: &[
("M", 6),
("B", 9),
],
},
@ -1579,10 +1585,11 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(1, 0),
(2, 0),
],
entries: &[
("G", 9),
("M", 6),
("Md", 9),
],
},
@ -1650,8 +1657,10 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
],
entries: &[
("M", 6),
],
},
},
@ -1924,13 +1933,14 @@ pub fn entry(lang: Language) -> Entry {
},
comma_decimal: true,
number_tokens: ::phf::Map {
key: 12913932095322966823,
key: 15467950696543387533,
disps: &[
(0, 0),
(2, 0),
],
entries: &[
("Mrd", 9),
("E", 3),
("Mrd", 9),
("M", 6),
],
},
},
@ -2051,12 +2061,13 @@ pub fn entry(lang: Language) -> Entry {
},
comma_decimal: true,
number_tokens: ::phf::Map {
key: 12913932095322966823,
key: 15467950696543387533,
disps: &[
(0, 0),
],
entries: &[
("jt", 6),
("M", 9),
("rb", 3),
],
},
@ -3820,11 +3831,12 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
(2, 0),
],
entries: &[
("mil", 3),
("mM", 9),
("M", 6),
("mil", 3),
],
},
},
@ -4497,11 +4509,12 @@ pub fn entry(lang: Language) -> Entry {
number_tokens: ::phf::Map {
key: 12913932095322966823,
disps: &[
(0, 0),
(1, 0),
],
entries: &[
("elfu", 3),
("B", 9),
("elfu", 3),
("M", 6),
],
},
},
@ -5136,6 +5149,7 @@ pub fn entry(lang: Language) -> Entry {
(0, 0),
],
entries: &[
("M", 6),
("B", 9),
],
},
@ -5254,6 +5268,7 @@ pub fn entry(lang: Language) -> Entry {
(0, 0),
],
entries: &[
("M", 6),
("B", 9),
],
},

87
src/util.rs
View file

@ -6,6 +6,8 @@ use once_cell::sync::Lazy;
use rand::Rng;
use url::Url;
use crate::{dictionary, model::Language};
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@ -228,8 +230,61 @@ impl<T> TryRemove<T> for Vec<T> {
}
}
fn parse_large_numstr(string: &str, lang: Language) -> Option<u64> {
let dict_entry = dictionary::entry(lang);
let decimal_point = match dict_entry.comma_decimal {
true => ',',
false => '.',
};
let (num, mut exp, filtered) = {
let mut buf = String::new();
let mut filtered = String::new();
let mut exp = 0;
let mut after_point = false;
for c in string.chars() {
if c.is_ascii_digit() {
buf.push(c);
if after_point {
exp -= 1;
}
} else if c == decimal_point {
after_point = true;
} else if !matches!(c, '\u{200b}' | '.' | ',') {
filtered.push(c);
}
}
(ok_or_bail!(buf.parse::<u64>(), None), exp, filtered)
};
let lookup_token = |token: &str| match token {
"K" | "k" => Some(3),
_ => dict_entry.number_tokens.get(token).map(|t| *t as i32),
};
if dict_entry.by_char {
exp += filtered
.chars()
.filter_map(|token| lookup_token(&token.to_string()))
.sum::<i32>();
} else {
exp += filtered
.split_whitespace()
.filter_map(lookup_token)
.sum::<i32>();
}
num.checked_mul(some_or_bail!(
(10_u64).checked_pow(ok_or_bail!(exp.try_into(), None)),
None
))
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader, path::Path};
use super::*;
use rstest::rstest;
@ -313,4 +368,36 @@ mod tests {
let res = sanitize_yt_url(url);
assert_eq!(res, expect);
}
#[test]
fn t_parse_large_numstr_samples() {
let json_path = Path::new("testfiles/dict/large_number_samples.json");
let json_file = File::open(json_path).unwrap();
let number_samples: BTreeMap<Language, BTreeMap<u8, (String, u64)>> =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
number_samples.iter().for_each(|(lang, entry)| {
entry.iter().for_each(|(_, (txt, expect))| {
testcase_parse_large_numstr(txt, *lang, *expect);
});
});
}
fn testcase_parse_large_numstr(string: &str, lang: Language, expect: u64) {
// Round the expected number to the amount of significant digits included
// in the string.
let rounded = {
let n_significant_d = string.chars().filter(char::is_ascii_digit).count();
let mag = (expect as f64).log10().floor();
let factor = 10_u64.pow(1 + mag as u32 - n_significant_d as u32);
(((expect as f64) / factor as f64).floor() as u64) * factor
};
let res = parse_large_numstr(string, lang).expect(string);
assert_eq!(
res, rounded,
"{} (lang: {}, exact: {})",
string, lang, expect
);
}
}