Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: use fancy-regex only for backtracking

Browse source
This commit is contained in:
ThetaDev 2023-01-23 15:33:05 +01:00
parent 4cc069fba2
commit 92a358a079
12 changed files with 69 additions and 108 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -26,6 +26,7 @@ rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
# quick-js = "0.4.1"
quick-js = { path = "../quickjs-rs", default-features = false }
once_cell = "1.12.0"
regex = "1.6.0"
fancy-regex = "0.11.0"
thiserror = "1.0.36"
url = "2.2.2"

2
src/client/mod.rs
View file

@ -25,10 +25,10 @@ mod channel_rss;
use std::sync::Arc;
use std::{borrow::Cow, fmt::Debug};
use fancy_regex::Regex;
use log::{debug, error, warn};
use once_cell::sync::Lazy;
use rand::Rng;
use regex::Regex;
use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use time::{Duration, OffsetDateTime};

4
src/client/music_artist.rs
View file

@ -1,8 +1,8 @@
use std::{borrow::Cow, rc::Rc};
use fancy_regex::Regex;
use futures::{stream, StreamExt};
use once_cell::sync::Lazy;
use regex::Regex;
use serde::Serialize;
use crate::{
@ -268,8 +268,6 @@ fn map_artist_page(
let wikipedia_url = header.description.as_deref().and_then(|h| {
WIKIPEDIA_REGEX
.captures(h)
.ok()
.flatten()
.and_then(|c| c.get(0))
.map(|m| m.as_str().to_owned())
});

6
src/client/player.rs
View file

@ -3,8 +3,8 @@ use std::{
collections::{BTreeMap, HashMap},
};
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::Serialize;
use url::Url;
@ -530,8 +530,6 @@ fn map_audio_stream(
Some(t) => {
let lang = LANG_PATTERN
.captures(&t.id)
.ok()
.flatten()
.map(|m| m.get(1).unwrap().as_str().to_owned());
Some(AudioTrack {
@ -557,7 +555,7 @@ fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> {
static PATTERN: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap());
let captures = some_or_bail!(PATTERN.captures(mime).ok().flatten(), None);
let captures = some_or_bail!(PATTERN.captures(mime), None);
Some((
captures.get(1).unwrap().as_str(),
captures

2
src/client/response/channel_rss.rs
View file

@ -97,7 +97,7 @@ impl From<ChannelRss> for crate::model::ChannelRss {
.uri
.strip_prefix("https://www.youtube.com/channel/")
.and_then(|id| {
if util::CHANNEL_ID_REGEX.is_match(id).unwrap_or_default() {
if util::CHANNEL_ID_REGEX.is_match(id) {
Some(id.to_owned())
} else {
None

18
src/client/response/video_item.rs
View file

@ -1,5 +1,5 @@
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::Deserialize;
use serde_with::{
json::JsonString, rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError,
@ -503,20 +503,12 @@ impl<T> YouTubeListMapper<T> {
id: video.video_id,
name: video.headline,
length: video.accessibility.and_then(|acc| {
ACCESSIBILITY_SEP_REGEX
.captures(&acc)
.ok()
.flatten()
.and_then(|cap| {
cap.get(1).and_then(|c| {
timeago::parse_timeago_or_warn(
self.lang,
c.as_str(),
&mut self.warnings,
)
ACCESSIBILITY_SEP_REGEX.captures(&acc).and_then(|cap| {
cap.get(1).and_then(|c| {
timeago::parse_timeago_or_warn(self.lang, c.as_str(), &mut self.warnings)
.map(|ta| Duration::from(ta).whole_seconds() as u32)
})
})
})
}),
thumbnail: video.thumbnail.into(),
channel: self.channel.clone(),

23
src/client/url_resolver.rs
View file

@ -112,9 +112,9 @@ impl RustyPipeQuery {
// Album or channel
Some("browse") => match path_split.next() {
Some(id) => {
if util::CHANNEL_ID_REGEX.is_match(id).unwrap_or_default() {
if util::CHANNEL_ID_REGEX.is_match(id) {
Ok(UrlTarget::Channel { id: id.to_owned() })
} else if util::ALBUM_ID_REGEX.is_match(id).unwrap_or_default() {
} else if util::ALBUM_ID_REGEX.is_match(id) {
Ok(UrlTarget::Album { id: id.to_owned() })
} else {
Err(Error::Other("invalid url: no browse id".into()))
@ -153,10 +153,7 @@ impl RustyPipeQuery {
// If there is a timestamp parameter, it has to be a video
// First check the innertube API if this is a channel vanity url
// If no channel is found and the identifier has the video ID format, assume it is a video
if !params.contains_key("t")
&& util::VANITY_PATH_REGEX
.is_match(url.path())
.unwrap_or_default()
if !params.contains_key("t") && util::VANITY_PATH_REGEX.is_match(url.path())
{
match self
._navigation_resolve_url(url.path(), ClientType::Desktop)
@ -164,7 +161,7 @@ impl RustyPipeQuery {
{
Ok(target) => Ok(target),
Err(Error::Extraction(ExtractionError::ContentUnavailable(e))) => {
match util::VIDEO_ID_REGEX.is_match(id).unwrap_or_default() {
match util::VIDEO_ID_REGEX.is_match(id) {
true => Ok(UrlTarget::Video {
id: id.to_owned(),
start_time: get_start_time(),
@ -176,7 +173,7 @@ impl RustyPipeQuery {
}
Err(e) => Err(e),
}
} else if util::VIDEO_ID_REGEX.is_match(id).unwrap_or_default() {
} else if util::VIDEO_ID_REGEX.is_match(id) {
Ok(UrlTarget::Video {
id: id.to_owned(),
start_time: get_start_time(),
@ -232,16 +229,16 @@ impl RustyPipeQuery {
.await
}
// ID only
else if util::VIDEO_ID_REGEX.is_match(string).unwrap_or_default() {
else if util::VIDEO_ID_REGEX.is_match(string) {
Ok(UrlTarget::Video {
id: string.to_owned(),
start_time: 0,
})
} else if util::CHANNEL_ID_REGEX.is_match(string).unwrap_or_default() {
} else if util::CHANNEL_ID_REGEX.is_match(string) {
Ok(UrlTarget::Channel {
id: string.to_owned(),
})
} else if util::PLAYLIST_ID_REGEX.is_match(string).unwrap_or_default() {
} else if util::PLAYLIST_ID_REGEX.is_match(string) {
if resolve_albums && string.starts_with(util::PLAYLIST_ID_ALBUM_PREFIX) {
self._navigation_resolve_url(
&format!("/playlist?list={}", string),
@ -253,13 +250,13 @@ impl RustyPipeQuery {
id: string.to_owned(),
})
}
} else if util::ALBUM_ID_REGEX.is_match(string).unwrap_or_default() {
} else if util::ALBUM_ID_REGEX.is_match(string) {
Ok(UrlTarget::Album {
id: string.to_owned(),
})
}
// Channel name only
else if util::VANITY_PATH_REGEX.is_match(string).unwrap_or_default() {
else if util::VANITY_PATH_REGEX.is_match(string) {
self._navigation_resolve_url(
&format!("/{}", string.trim_start_matches('/')),
ClientType::Desktop,

38
src/deobfuscate.rs
View file

@ -1,6 +1,7 @@
use fancy_regex::Regex;
use fancy_regex::Regex as FancyRegex;
use log::debug;
use once_cell::sync::Lazy;
use regex::Regex;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::result::Result::Ok;
@ -68,18 +69,18 @@ impl From<DeobfData> for Deobfuscator {
const DEOBFUSCATION_FUNC_NAME: &str = "deobfuscate";
fn get_sig_fn_name(player_js: &str) -> Result<String> {
static FUNCTION_REGEXES: Lazy<[Regex; 6]> = Lazy::new(|| {
static FUNCTION_REGEXES: Lazy<[FancyRegex; 6]> = Lazy::new(|| {
[
Regex::new("(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)").unwrap(),
Regex::new("\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)").unwrap(),
Regex::new("\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)").unwrap(),
Regex::new("([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;").unwrap(),
Regex::new("\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;").unwrap(),
Regex::new("\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(").unwrap(),
FancyRegex::new("(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)").unwrap(),
FancyRegex::new("\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)").unwrap(),
FancyRegex::new("\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)").unwrap(),
FancyRegex::new("([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;").unwrap(),
FancyRegex::new("\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(\"\"\\)\\s*;").unwrap(),
FancyRegex::new("\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(").unwrap(),
]
});
util::get_cg_from_regexes(FUNCTION_REGEXES.iter(), player_js, 1)
util::get_cg_from_fancy_regexes(FUNCTION_REGEXES.iter(), player_js, 1)
.ok_or(DeobfError::Extraction("deobf function name"))
}
@ -98,8 +99,6 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
let deobfuscate_function = "var ".to_owned()
+ function_pattern
.captures(player_js)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("deobf function"))?
.get(1)
.unwrap()
@ -110,8 +109,6 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
Lazy::new(|| Regex::new(";([A-Za-z0-9_\\$]{2})\\...\\(").unwrap());
let helper_object_name = HELPER_OBJECT_NAME_REGEX
.captures(&deobfuscate_function)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("helper object name"))?
.get(1)
.unwrap()
@ -124,8 +121,6 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
let player_js_nonl = player_js.replace('\n', "");
let helper_object = helper_pattern
.captures(&player_js_nonl)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("helper object"))?
.get(1)
.unwrap()
@ -154,8 +149,6 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
let fname_match = FUNCTION_NAME_REGEX
.captures(player_js)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("n_deobf function"))?;
let function_name = fname_match.get(1).unwrap().as_str();
@ -171,15 +164,13 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
.parse::<usize>()
.or(Err(DeobfError::Other("could not parse array_num")))?;
let array_pattern_str =
"var ".to_owned() + &fancy_regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];";
"var ".to_owned() + &regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];";
let array_pattern = Regex::new(&array_pattern_str).or(Err(DeobfError::Other(
"could not parse helper pattern regex",
)))?;
let array_str = array_pattern
.captures(player_js)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("n_deobf array_str"))?
.get(1)
.unwrap()
@ -274,13 +265,10 @@ async fn get_player_js_url(http: &Client) -> Result<String> {
let text = resp.text().await?;
static PLAYER_HASH_PATTERN: Lazy<Regex> = Lazy::new(|| {
Regex::new(r#"https:\\\/\\\/www\.youtube\.com\\\/s\\\/player\\\/([a-z0-9]{8})\\\/"#)
.unwrap()
Regex::new(r#"https:\\/\\/www\.youtube\.com\\/s\\/player\\/([a-z0-9]{8})\\/"#).unwrap()
});
let player_hash = PLAYER_HASH_PATTERN
.captures(&text)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("player hash"))?
.get(1)
.unwrap()
@ -303,8 +291,6 @@ fn get_sts(player_js: &str) -> Result<String> {
Ok(STS_PATTERN
.captures(player_js)
.ok()
.flatten()
.ok_or(DeobfError::Extraction("sts"))?
.get(1)
.unwrap()

40
src/model/mod.rs
View file

@ -111,30 +111,22 @@ impl UrlTarget {
/// Validate the YouTube ID from the URL target
pub(crate) fn validate(&self) -> Result<(), Error> {
match self {
UrlTarget::Video { id, .. } => {
match util::VIDEO_ID_REGEX.is_match(id).unwrap_or_default() {
true => Ok(()),
false => Err(Error::Other("invalid video id".into())),
}
}
UrlTarget::Channel { id } => {
match util::CHANNEL_ID_REGEX.is_match(id).unwrap_or_default() {
true => Ok(()),
false => Err(Error::Other("invalid channel id".into())),
}
}
UrlTarget::Playlist { id } => {
match util::PLAYLIST_ID_REGEX.is_match(id).unwrap_or_default() {
true => Ok(()),
false => Err(Error::Other("invalid playlist id".into())),
}
}
UrlTarget::Album { id } => {
match util::ALBUM_ID_REGEX.is_match(id).unwrap_or_default() {
true => Ok(()),
false => Err(Error::Other("invalid album id".into())),
}
}
UrlTarget::Video { id, .. } => match util::VIDEO_ID_REGEX.is_match(id) {
true => Ok(()),
false => Err(Error::Other("invalid video id".into())),
},
UrlTarget::Channel { id } => match util::CHANNEL_ID_REGEX.is_match(id) {
true => Ok(()),
false => Err(Error::Other("invalid channel id".into())),
},
UrlTarget::Playlist { id } => match util::PLAYLIST_ID_REGEX.is_match(id) {
true => Ok(()),
false => Err(Error::Other("invalid playlist id".into())),
},
UrlTarget::Album { id } => match util::ALBUM_ID_REGEX.is_match(id) {
true => Ok(()),
false => Err(Error::Other("invalid album id".into())),
},
}
}
}

2
src/serializer/text.rs
View file

@ -1,7 +1,7 @@
use std::convert::TryFrom;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs};

19
src/util/mod.rs
View file

@ -13,9 +13,10 @@ use std::{
};
use base64::Engine;
use fancy_regex::Regex;
use fancy_regex::Regex as FancyRegex;
use once_cell::sync::Lazy;
use rand::Rng;
use regex::Regex;
use url::Url;
use crate::{error::Error, param::Language};
@ -28,7 +29,7 @@ pub static PLAYLIST_ID_REGEX: Lazy<Regex> =
pub static ALBUM_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap());
pub static VANITY_PATH_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^/?(?:(?:c\/|user\/)?[A-z0-9]+)|(?:@[A-z0-9-_.]+)$").unwrap());
Lazy::new(|| Regex::new(r"^/?(?:(?:c/|user/)?[A-z0-9]+)|(?:@[A-z0-9-_.]+)$").unwrap());
/// Separator string for YouTube Music subtitles
pub const DOT_SEPARATOR: &str = " • ";
@ -49,6 +50,16 @@ pub struct MappingError(pub(crate) Cow<'static, str>);
pub fn get_cg_from_regexes<'a, I>(mut regexes: I, text: &str, cg: usize) -> Option<String>
where
I: Iterator<Item = &'a Regex>,
{
regexes
.find_map(|pattern| pattern.captures(text))
.map(|c| c.get(cg).unwrap().as_str().to_owned())
}
/// Return the given capture group that matches first in a list of fancy regexes
pub fn get_cg_from_fancy_regexes<'a, I>(mut regexes: I, text: &str, cg: usize) -> Option<String>
where
I: Iterator<Item = &'a FancyRegex>,
{
regexes
.find_map(|pattern| pattern.captures(text).ok().flatten())
@ -132,7 +143,7 @@ where
pub fn parse_video_length(text: &str) -> Option<u32> {
static VIDEO_LENGTH_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"(?:(\d+):)?(\d{1,2}):(\d{2})"#).unwrap());
VIDEO_LENGTH_REGEX.captures(text).ok().flatten().map(|cap| {
VIDEO_LENGTH_REGEX.captures(text).map(|cap| {
let hrs = cap
.get(1)
.and_then(|x| x.as_str().parse::<u32>().ok())
@ -339,8 +350,6 @@ pub fn video_id_from_thumbnail_url(url: &str) -> Option<String> {
Lazy::new(|| Regex::new(r"^https://i.ytimg.com/vi/([A-Za-z0-9_-]{11})/").unwrap());
URL_REGEX
.captures(url)
.ok()
.flatten()
.and_then(|cap| cap.get(1).map(|x| x.as_str().to_owned()))
}

22
tests/youtube.rs
View file

@ -1,8 +1,8 @@
use std::collections::HashSet;
use std::fmt::Display;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use regex::Regex;
use rstest::rstest;
use time::macros::date;
use time::OffsetDateTime;
@ -2272,11 +2272,7 @@ async fn assert_next_items<T: FromYtItem, Q: AsRef<RustyPipeQuery>>(
fn assert_video_id(id: &str) {
static VIDEO_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^[A-Za-z0-9_-]{11}$").unwrap());
assert!(
VIDEO_ID_REGEX.is_match(id).unwrap_or_default(),
"invalid video id: `{}`",
id
);
assert!(VIDEO_ID_REGEX.is_match(id), "invalid video id: `{}`", id);
}
fn assert_channel_id(id: &str) {
@ -2284,7 +2280,7 @@ fn assert_channel_id(id: &str) {
Lazy::new(|| Regex::new(r"^UC[A-Za-z0-9_-]{22}$").unwrap());
assert!(
CHANNEL_ID_REGEX.is_match(id).unwrap_or_default(),
CHANNEL_ID_REGEX.is_match(id),
"invalid channel id: `{}`",
id
);
@ -2294,20 +2290,12 @@ fn assert_album_id(id: &str) {
static ALBUM_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^MPREb_[A-Za-z0-9_-]{11}$").unwrap());
assert!(
ALBUM_ID_REGEX.is_match(id).unwrap_or_default(),
"invalid album id: `{}`",
id
);
assert!(ALBUM_ID_REGEX.is_match(id), "invalid album id: `{}`", id);
}
fn assert_playlist_id(id: &str) {
static PLAYLIST_ID_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r"^(?:PL|RD|OLAK)[A-Za-z0-9_-]{30,}$").unwrap());
assert!(
PLAYLIST_ID_REGEX.is_match(id).unwrap_or_default(),
"invalid album id: `{}`",
id
);
assert!(PLAYLIST_ID_REGEX.is_match(id), "invalid album id: `{}`", id);
}