mod date; mod protobuf; pub mod dictionary; pub mod timeago; pub use date::{now_sec, shift_months, shift_years}; pub use protobuf::{string_from_pb, ProtoBuilder}; use std::{ borrow::{Borrow, Cow}, collections::BTreeMap, str::{FromStr, SplitWhitespace}, }; use base64::Engine; use fancy_regex::Regex as FancyRegex; use once_cell::sync::Lazy; use rand::Rng; use regex::Regex; use url::Url; use crate::{error::Error, param::Language, serializer::text::TextComponent}; pub static VIDEO_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-]{11}$").unwrap()); pub static CHANNEL_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap()); pub static PLAYLIST_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^(?:PL|RDCLAK|OLAK)[A-Za-z0-9_-]{16,50}$").unwrap()); pub static ALBUM_ID_REGEX: Lazy = Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap()); pub static VANITY_PATH_REGEX: Lazy = Lazy::new(|| { Regex::new(r"^/?(?:(?:c/|user/)?[A-z0-9]{1,100})|(?:@[A-z0-9-_.]{1,100})$").unwrap() }); /// Separator string for YouTube Music subtitles pub const DOT_SEPARATOR: &str = " • "; pub const VARIOUS_ARTISTS: &str = "Various Artists"; pub const PLAYLIST_ID_ALBUM_PREFIX: &str = "OLAK"; const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; /// Internal error #[derive(thiserror::Error, Debug)] #[error("mapping error: {0}")] pub struct MappingError(pub(crate) Cow<'static, str>); /// Return the given capture group that matches first in a list of regexes pub fn get_cg_from_regexes<'a, I>(mut regexes: I, text: &str, cg: usize) -> Option where I: Iterator, { regexes .find_map(|pattern| pattern.captures(text)) .map(|c| c.get(cg).unwrap().as_str().to_owned()) } /// Return the given capture group that matches first in a list of fancy regexes pub fn get_cg_from_fancy_regexes<'a, I>(mut regexes: I, text: &str, cg: usize) -> Option where I: Iterator, { regexes .find_map(|pattern| pattern.captures(text).ok().flatten()) .map(|c| c.get(cg).unwrap().as_str().to_owned()) } /// Generate a random string with given length and byte charset. fn random_string(charset: &[u8], length: usize) -> String { let mut result = String::with_capacity(length); let mut rng = rand::thread_rng(); for _ in 0..length { result.push(char::from(charset[rng.gen_range(0..charset.len())])); } result } /// Generate a 16 characters long random string used as a CPN (Content Playback Nonce) pub fn generate_content_playback_nonce() -> String { random_string(CONTENT_PLAYBACK_NONCE_ALPHABET, 16) } pub fn random_uuid() -> String { let mut rng = rand::thread_rng(); format!( "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", rng.gen::(), rng.gen::(), rng.gen::(), rng.gen::(), rng.gen::() & 0xffffffffffff, ) } /// Split an URL into its base string and parameter map /// /// Example: /// /// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap), Error> { let mut parsed_url = Url::parse(url) .map_err(|e| Error::Other(format!("could not parse url `{url}` err: {e}").into()))?; let url_params: BTreeMap = parsed_url .query_pairs() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); parsed_url.set_query(None); Ok((parsed_url, url_params)) } /// Parse a string after removing all non-numeric characters pub fn parse_numeric(string: &str) -> Result where F: FromStr, { let mut buf = String::new(); for c in string.chars() { if c.is_ascii_digit() { buf.push(c); } } buf.parse() } /// Parse all numbers occurring in a string and reurn them as a vec pub fn parse_numeric_vec(string: &str) -> Vec where F: FromStr, { let mut numbers = vec![]; let mut buf = String::new(); for c in string.chars() { if c.is_ascii_digit() { buf.push(c); } else if !buf.is_empty() { buf.parse::().map_or((), |n| numbers.push(n)); buf.clear(); } } if !buf.is_empty() { buf.parse::().map_or((), |n| numbers.push(n)); } numbers } /// Parse textual video length (e.g. `0:49`, `2:02` or `1:48:18`) /// and return the duration in seconds. pub fn parse_video_length(text: &str) -> Option { static VIDEO_LENGTH_REGEX: Lazy = Lazy::new(|| Regex::new(r#"(?:(\d+)[:.])?(\d{1,2})[:.](\d{2})"#).unwrap()); VIDEO_LENGTH_REGEX.captures(text).map(|cap| { let hrs = cap .get(1) .and_then(|x| x.as_str().parse::().ok()) .unwrap_or_default(); let min = cap .get(2) .and_then(|x| x.as_str().parse::().ok()) .unwrap_or_default(); let sec = cap .get(3) .and_then(|x| x.as_str().parse::().ok()) .unwrap_or_default(); hrs * 3600 + min * 60 + sec }) } pub fn parse_numeric_or_warn(string: &str, warnings: &mut Vec) -> Option where F: FromStr, { let res = parse_numeric::(string); if res.is_err() { warnings.push(format!("could not parse number `{string}`")); } res.ok() } pub fn retry_delay( n_past_retries: u32, min_retry_interval: u32, max_retry_interval: u32, backoff_base: u32, ) -> u32 { let unjittered_delay = backoff_base.checked_pow(n_past_retries).unwrap_or(u32::MAX); let jitter_factor = rand::thread_rng().gen_range(800..1500); let jittered_delay = unjittered_delay .checked_mul(jitter_factor) .unwrap_or(u32::MAX); min_retry_interval.max(jittered_delay.min(max_retry_interval)) } /// Convert YouTube redirect URLs (`https://www.youtube.com/redirect?`) into regular URLs. /// /// Also strips google analytics tracking parameters /// (`utm_source`, `utm_medium`, `utm_campaign`, `utm_content`) because google analytics is bad. pub fn sanitize_yt_url(url: &str) -> String { fn sanitize_yt_url_inner(url: &str) -> Option { let mut parsed_url = Url::parse(url).ok()?; // Convert redirect url if parsed_url.host_str().unwrap_or_default() == "www.youtube.com" && parsed_url.path() == "/redirect" { if let Some((_, url)) = parsed_url.query_pairs().find(|(k, _)| k == "q") { parsed_url = Url::parse(url.as_ref()).ok()?; } } // Remove GA tracking params if parsed_url.query().is_some() { let params = parsed_url .query_pairs() .filter_map(|(k, v)| match k.borrow() { "utm_source" | "utm_medium" | "utm_campaign" | "utm_content" => None, _ => Some((k.to_string(), v.to_string())), }) .collect::>(); // Set empty query string if there are no parameters to prevent urls from ending with /? if params.is_empty() { parsed_url.set_query(None); } else { parsed_url .query_pairs_mut() .clear() .extend_pairs(params) .finish(); } } Some(parsed_url.to_string()) } sanitize_yt_url_inner(url).unwrap_or_else(|| url.to_string()) } pub trait TryRemove { /// Removes and returns the element at position `index` within the vector, /// shifting all elements after it to the left. /// /// Returns None if the index is out of bounds. /// /// Note: Because this shifts over the remaining elements, it has a /// worst-case performance of *O*(*n*). If you don't need the order of elements /// to be preserved, use [`vec_try_swap_remove`] instead. fn try_remove(&mut self, index: usize) -> Option; /// Removes an element from the vector and returns it. /// /// The removed element is replaced by the last element of the vector. /// /// Returns None if the index is out of bounds. /// /// This does not preserve ordering, but is *O*(1). /// If you need to preserve the element order, use [`vec_try_remove`] instead. fn try_swap_remove(&mut self, index: usize) -> Option; } impl TryRemove for Vec { fn try_remove(&mut self, index: usize) -> Option { if index < self.len() { Some(self.remove(index)) } else { None } } fn try_swap_remove(&mut self, index: usize) -> Option { if index < self.len() { Some(self.swap_remove(index)) } else { None } } } /// Check if a channel name equals "YouTube Music" /// (the author of original YouTube music playlists) pub(crate) fn is_ytm(text: &TextComponent) -> bool { if let TextComponent::Text { text } = text { text.starts_with("YouTube") } else { false } } /// Check if a language should be parsed by character pub fn lang_by_char(lang: Language) -> bool { matches!( lang, Language::Ja | Language::ZhCn | Language::ZhHk | Language::ZhTw ) } /// Parse a large, textual number (e.g. `1.4M subscribers`, `22K views`) pub fn parse_large_numstr(string: &str, lang: Language) -> Option where F: TryFrom, { // Special case for Gujarati: the "no views" text does not contain // any parseable tokens: the 2 words occur in any view count text. // This may be a translation error. if lang == Language::Gu && string == "જોવાયાની સંખ્યા" { return 0.try_into().ok(); } let dict_entry = dictionary::entry(lang); let by_char = lang_by_char(lang) || lang == Language::Ko; let decimal_point = match dict_entry.comma_decimal { true => ',', false => '.', }; let mut digits = String::new(); let mut filtered = String::new(); let mut exp = 0; let mut after_point = false; for c in string.chars() { if c.is_ascii_digit() { digits.push(c); if after_point { exp -= 1; } } else if c == decimal_point { after_point = true; } else if !matches!( c, '\u{200b}' | '\u{202b}' | '\u{202c}' | '\u{202e}' | '\u{200e}' | '\u{200f}' | '.' | ',' ) { c.to_lowercase().for_each(|c| filtered.push(c)); } } if digits.is_empty() { SplitTokens::new(&filtered, by_char) .find_map(|token| dict_entry.number_nd_tokens.get(token)) .and_then(|n| (*n as u64).try_into().ok()) } else { let num = digits.parse::().ok()?; exp += SplitTokens::new(&filtered, by_char) .filter_map(|token| match token { "k" => Some(3), _ => dict_entry.number_tokens.get(token).map(|t| *t as i32), }) .sum::(); F::try_from(num.checked_mul((10_u64).checked_pow(exp.try_into().ok()?)?)?).ok() } } pub fn parse_large_numstr_or_warn( string: &str, lang: Language, warnings: &mut Vec, ) -> Option where F: TryFrom, { let res = parse_large_numstr::(string, lang); if res.is_none() { warnings.push(format!("could not parse numstr `{string}`")); } res } /// Replace all html control characters to make a string safe for inserting into HTML. pub fn escape_html(input: &str) -> String { let mut buf = String::with_capacity(input.len()); for c in input.chars() { match c { '<' => buf.push_str("<"), '>' => buf.push_str(">"), '&' => buf.push_str("&"), '"' => buf.push_str("""), '\'' => buf.push_str("'"), '\n' => buf.push_str("
"), _ => buf.push(c), }; } buf } pub fn video_id_from_thumbnail_url(url: &str) -> Option { static URL_REGEX: Lazy = Lazy::new(|| Regex::new(r"^https://i.ytimg.com/vi/([A-Za-z0-9_-]{11})/").unwrap()); URL_REGEX .captures(url) .and_then(|cap| cap.get(1).map(|x| x.as_str().to_owned())) } pub fn b64_encode>(input: T) -> String { base64::engine::general_purpose::STANDARD.encode(input) } pub fn b64_decode>(input: T) -> Result, base64::DecodeError> { base64::engine::general_purpose::STANDARD.decode(input) } /// An iterator over the chars in a string (in str format) pub struct SplitChar<'a> { txt: &'a str, index: usize, } impl<'a> From<&'a str> for SplitChar<'a> { fn from(value: &'a str) -> Self { Self { txt: value, index: 0, } } } impl<'a> Iterator for SplitChar<'a> { type Item = &'a str; fn next(&mut self) -> Option { self.txt .get(self.index..) .and_then(|txt| txt.chars().next()) .map(|c| { let start = self.index; self.index += c.len_utf8(); &self.txt[start..self.index] }) } } /// An iterator for parsing strings. It can either iterate over words or characters. pub enum SplitTokens<'a> { Word(SplitWhitespace<'a>), Char(SplitChar<'a>), } impl<'a> SplitTokens<'a> { pub fn new(s: &'a str, by_char: bool) -> Self { match by_char { true => Self::Char(SplitChar::from(s)), false => Self::Word(s.split_whitespace()), } } } impl<'a> Iterator for SplitTokens<'a> { type Item = &'a str; fn next(&mut self) -> Option { match self { SplitTokens::Word(iter) => iter.next(), SplitTokens::Char(iter) => iter.next(), } } } #[cfg(test)] pub(crate) mod tests { use std::{fs::File, io::BufReader, path::PathBuf}; use path_macro::path; use rstest::rstest; use super::*; /// Get the path of the `testfiles` directory pub static TESTFILES: Lazy = Lazy::new(|| path!(env!("CARGO_MANIFEST_DIR") / "testfiles")); #[rstest] #[case("1.000", 1000)] #[case("4 Hello World 2", 42)] fn t_parse_num(#[case] string: &str, #[case] expect: u32) { let n = parse_numeric::(string).unwrap(); assert_eq!(n, expect); } #[rstest] #[case("15.03.2022", vec![15, 3, 2022])] #[case("4 Hello World 2", vec![4, 2])] #[case("最后更新时间:2020年1月3日", vec![2020, 1, 3])] fn t_parse_numeric_vec(#[case] string: &str, #[case] expect: Vec) { let n = parse_numeric_vec::(string); assert_eq!(n, expect); } #[rstest] #[case("0:49", Some(49))] #[case("bla 2:02 h3llo w0rld", Some(122))] #[case("18:22", Some(1102))] #[case("1:48:18", Some(6498))] #[case("102:12:39", Some(367959))] #[case("42", None)] fn t_parse_video_length(#[case] text: &str, #[case] expect: Option) { let n = parse_video_length(text); assert_eq!(n, expect); } #[rstest] #[case(0, 800, 1500)] #[case(1, 2400, 4500)] #[case(2, 7200, 13500)] #[case(100, 60000, 60000)] fn t_retry_delay(#[case] n: u32, #[case] expect_min: u32, #[case] expect_max: u32) { let res = retry_delay(n, 1000, 60000, 3); assert!( res >= expect_min && res <= expect_max, "res: {res} not within {expect_min} and {expect_max}" ); } #[test] fn t_vec_try_remove() { let mut v = vec![1, 2, 3]; assert_eq!(v.try_remove(0).unwrap(), 1); assert_eq!(v.try_remove(1).unwrap(), 3); assert_eq!(v.try_remove(1), None); } #[test] fn t_vec_try_swap_remove() { let mut v = vec![1, 2, 3]; assert_eq!(v.try_swap_remove(0).unwrap(), 1); assert_eq!(v.try_swap_remove(1).unwrap(), 2); assert_eq!(v.try_swap_remove(1), None); } #[rstest] #[case( "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbXFjbjZ6bWdHc1VFLVNBN1NiRGR1QmRuR0lGZ3xBQ3Jtc0trcG1fWHpRNlE2eGNER0ZGczFlZXM5ZlctZzFSbl8wcHdieTlTb1ktSUc5OTZxVDVQamcxdS0yRjJJelFWTGdOS09nUk8xRExqbWhOSG5MTm83WG1QQzJqZTJuT2d6cGp0cEZTWmdsal80ODk0WkNESQ&q=http%3A%2F%2Fincompetech.com%2Fmusic%2Froyalty-free%2F&v=86YLFOog4GM", "http://incompetech.com/music/royalty-free/", )] #[case("https://www.gnu.org", "https://www.gnu.org/")] #[case( "https://www.youtube.com/watch?v=Rp2V7d69hyM", "https://www.youtube.com/watch?v=Rp2V7d69hyM" )] #[case( "https://www.youtube.com/redirect?event=product_shelf&redir_token=QUFFLUhqbDVUMUF3SndkcDFJbzMxYkNIMDRWSzRVQU84QXxBQ3Jtc0tsQWdpaUlaMzFUQmQwSGYwR3dDRDhHWld1bFFtUmlmMng0MmxtN19iVW1EeV9oSk1Xb1VlQ1UyT2xUOWhPdUZvVEZ6UWE4Unlia3pwZXhpUmd4RVg4eWZtcHFId2RJVkMyMUFIMDhiUVUzc2x6ZVNxbw&q=https%3A%2F%2Flttstore.com%2F%3Futm_medium%3Dproduct_shelf%26utm_source%3Dyoutube%26utm_content%3DYT-AERwsnLS3vZeiqL7_mR16DPg7FPBWvP7OW-zX2M1UIPlexPS8-gpk-2c3epSZ8lJ5NYbLof0MXDKhRLCSyfOn9BYJrcG8YtpTA9VU2VXUVhhl9AKi87G_-vFhj6jcGN1CWcYYvmZYbIqA93kwkeFuUh46ntDZR1Y8p5WygwVlhfxy_BZiNbzkWw%253D&v=nFDBxBUfE74", "https://lttstore.com/", )] fn t_sanitize_yt_url(#[case] url: &str, #[case] expect: &str) { let res = sanitize_yt_url(url); assert_eq!(res, expect); } #[rstest] #[case( Language::Iw, "\u{200f}\u{202b}3.36M\u{200f}\u{202c}\u{200f} \u{200f}מנויים\u{200f}", 3_360_000 )] #[case(Language::As, "১ জন গ্ৰাহক", 1)] fn t_parse_large_numstr(#[case] lang: Language, #[case] string: &str, #[case] expect: u64) { let res = parse_large_numstr::(string, lang).unwrap(); assert_eq!(res, expect); } #[test] fn t_parse_large_numstr_samples() { let json_path = path!(*TESTFILES / "dict" / "large_number_samples.json"); let json_file = File::open(json_path).unwrap(); let number_samples: BTreeMap> = serde_json::from_reader(BufReader::new(json_file)).unwrap(); number_samples.iter().for_each(|(lang, entry)| { entry.iter().for_each(|(txt, expect)| { testcase_parse_large_numstr(txt, *lang, *expect); }); }); } fn testcase_parse_large_numstr(string: &str, lang: Language, expect: u64) { // Round the expected number to the amount of significant digits included // in the string. let rounded = { let n_significant_d = string.chars().filter(char::is_ascii_digit).count(); if n_significant_d == 0 { expect } else { let mag = (expect as f64).log10().floor(); let factor = 10_u64.pow(1 + mag as u32 - n_significant_d as u32); (((expect as f64) / factor as f64).floor() as u64) * factor } }; let emsg = format!("{string} (lang: {lang}, exact: {expect})"); let res = parse_large_numstr::(string, lang).expect(&emsg); assert_eq!(res, rounded, "{emsg}"); } #[test] fn split_char() { let teststr = "abc今天更新def"; let res = SplitTokens::new(teststr, true).collect::>(); assert_eq!(res.len(), 10); let res_str = res.into_iter().collect::(); assert_eq!(res_str, teststr) } #[test] fn split_words() { let teststr = "abc 今天更新 ghi"; let res = SplitTokens::new(teststr, false).collect::>(); assert_eq!(res.len(), 3); let res_str = res.join(" "); assert_eq!(res_str, teststr) } #[rstest] #[case("en", Some(Language::En))] #[case("en-GB", Some(Language::EnGb))] #[case("en-US", Some(Language::En))] #[case("en-ZZ", Some(Language::En))] #[case("xy", None)] #[case("xy-ZZ", None)] fn parse_language(#[case] s: &str, #[case] expect: Option) { let res = Language::from_str(s).ok(); assert_eq!(res, expect); } }