diff --git a/.gitignore b/.gitignore index cb98b06..e96be3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /target /Cargo.lock -RustyPipeReports -RustyPipeCache.json -rusty-tube.json +rustypipe_reports +rustypipe_cache.json diff --git a/Cargo.toml b/Cargo.toml index b529a84..2e4e6f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,12 +7,16 @@ edition = "2021" members = [".", "cli"] [features] -default = ["default-tls"] +default = ["default-tls", "yaml"] +# Reqwest TLS default-tls = ["reqwest/default-tls"] rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] +# Error reports in yaml format +yaml = ["serde_yaml"] + [dependencies] # quick-js = "0.4.1" quick-js = { path = "../quickjs-rs" } @@ -26,10 +30,9 @@ reqwest = {version = "0.11.11", default-features = false, features = ["json", "g tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" -serde_yaml = "0.9.11" +serde_yaml = {version = "0.9.11", optional = true} serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" -async-trait = "0.1.56" chrono = {version = "0.4.19", features = ["serde"]} chronoutil = "0.2.3" futures = "0.3.21" diff --git a/src/cache.rs b/src/cache.rs index 509f34b..e3339d3 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,364 +1,57 @@ use std::{ - fs::File, - future::Future, - io::BufReader, + fs, path::{Path, PathBuf}, - sync::Arc, }; -use anyhow::Result; -use chrono::{DateTime, Duration, Utc}; -use log::{error, info}; -use serde::{Deserialize, Serialize}; -use tokio::sync::Mutex; +use log::error; -#[derive(Default, Debug, Clone)] -pub struct Cache { - file: Option, - data: Arc>, +pub trait CacheStorage { + fn write(&self, data: &str); + fn read(&self) -> Option; } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -struct CacheData { - desktop_client: Option>, - music_client: Option>, - deobf: Option>, +pub struct FileStorage { + path: PathBuf, } -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CacheEntry { - last_update: DateTime, - data: T, -} - -impl From for CacheEntry { - fn from(f: T) -> Self { +impl FileStorage { + pub fn new>(path: P) -> Self { Self { - last_update: Utc::now(), - data: f, + path: path.as_ref().to_path_buf(), } } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ClientData { - pub version: String, +impl Default for FileStorage { + fn default() -> Self { + Self { + path: Path::new("rustypipe_cache.json").into(), + } + } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DeobfData { - pub js_url: String, - pub sig_fn: String, - pub nsig_fn: String, - pub sts: String, -} - -impl Cache { - pub async fn get_desktop_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.desktop_client.is_none() - || cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.desktop_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.desktop_client.as_ref().unwrap().data.clone()) - } +impl CacheStorage for FileStorage { + fn write(&self, data: &str) { + fs::write(&self.path, data).unwrap_or_else(|e| { + error!( + "Could not write cache to file `{}`. Error: {}", + self.path.to_string_lossy(), + e + ); + }); } - pub async fn get_music_client_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - - if cache.music_client.is_none() - || cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let cdata = updater.await?; - cache.music_client = Some(CacheEntry::from(cdata.clone())); - self.save(&cache); - Ok(cdata) - } else { - Ok(cache.music_client.as_ref().unwrap().data.clone()) - } - } - - pub async fn get_deobf_data(&self, updater: F) -> Result - where - F: Future> + Send + 'static, - { - let mut cache = self.data.lock().await; - if cache.deobf.is_none() - || cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24) - { - let deobf_data = updater.await?; - cache.deobf = Some(CacheEntry::from(deobf_data.clone())); - self.save(&cache); - Ok(deobf_data) - } else { - Ok(cache.deobf.as_ref().unwrap().data.clone()) - } - } - - pub async fn to_json(&self) -> Result { - let cache = self.data.lock().await; - Ok(serde_json::to_string(&cache.clone())?) - } - - pub async fn to_json_file>(&self, path: P) -> Result<()> { - let cache = self.data.lock().await; - Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?) - } - - pub fn from_json(json: &str) -> Self { - let data: CacheData = match serde_json::from_str(json) { - Ok(cd) => cd, + fn read(&self) -> Option { + match fs::read_to_string(&self.path) { + Ok(data) => Some(data), Err(e) => { error!( - "Could not load cache from json, falling back to default. Error: {}", + "Could not load cache from file `{}`. Error: {}", + self.path.to_string_lossy(), e ); - CacheData::default() + None } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: None, - } - } - - pub fn from_json_file>(path: P) -> Self { - let file = match File::open(path.as_ref()) { - Ok(file) => file, - Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - info!( - "Cache json file at {} not found, will be created", - path.as_ref().to_string_lossy() - ) - } else { - error!( - "Could not open cache json file, falling back to default. Error: {}", - e - ); - } - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - let data: CacheData = match serde_json::from_reader(BufReader::new(file)) { - Ok(data) => data, - Err(e) => { - error!( - "Could not load cache from json, falling back to default. Error: {}", - e - ); - return Cache { - file: Some(path.as_ref().to_path_buf()), - ..Default::default() - }; - } - }; - Cache { - data: Arc::new(Mutex::new(data)), - file: Some(path.as_ref().to_path_buf()), - } - } - - fn save(&self, cache: &CacheData) { - match self.file.as_ref() { - Some(file) => match File::create(file) { - Ok(file) => match serde_json::to_writer(file, cache) { - Ok(_) => {} - Err(e) => error!("Could not write cache to json. Error: {}", e), - }, - Err(e) => error!("Could not open cache json file. Error: {}", e), - }, - None => {} } } } - -#[cfg(test)] -mod tests { - use temp_testdir::TempDir; - - use super::*; - - #[tokio::test] - async fn test() { - let cache = Cache::default(); - - let desktop_c = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - desktop_c, - ClientData { - version: "1.2.3".to_owned() - } - ); - - let music_c = cache - .get_music_client_data(async { - Ok(ClientData { - version: "4.5.6".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - music_c, - ClientData { - version: "4.5.6".to_owned() - } - ); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert_eq!( - deobf_data, - DeobfData { - js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - } - ); - - // Create a new cache from the first one's json - // and check if it returns the same cached data - let json = cache.to_json().await.unwrap(); - let new_cache = Cache::from_json(&json); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - desktop_c - ); - - assert_eq!( - new_cache - .get_music_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - music_c - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } - - #[tokio::test] - async fn test_file() { - let temp = TempDir::default(); - let mut file_path = PathBuf::from(temp.as_ref()); - file_path.push("cache.json"); - - let cache = Cache::from_json_file(file_path.clone()); - - let cdata = cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "1.2.3".to_owned(), - }) - }) - .await - .unwrap(); - - let deobf_data = cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: - "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js" - .to_owned(), - sig_fn: "t_sig_fn".to_owned(), - nsig_fn: "t_nsig_fn".to_owned(), - sts: "t_sts".to_owned(), - }) - }) - .await - .unwrap(); - - assert!(file_path.exists()); - let new_cache = Cache::from_json_file(file_path.clone()); - - assert_eq!( - new_cache - .get_desktop_client_data(async { - Ok(ClientData { - version: "".to_owned(), - }) - }) - .await - .unwrap(), - cdata - ); - - assert_eq!( - new_cache - .get_deobf_data(async { - Ok(DeobfData { - js_url: "".to_owned(), - nsig_fn: "".to_owned(), - sig_fn: "".to_owned(), - sts: "".to_owned(), - }) - }) - .await - .unwrap(), - deobf_data - ); - } -} diff --git a/src/client2/mod.rs b/src/client2/mod.rs index 91b5622..356e59a 100644 --- a/src/client2/mod.rs +++ b/src/client2/mod.rs @@ -6,20 +6,35 @@ mod response; use std::fmt::Debug; use std::sync::Arc; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; +use chrono::{DateTime, Duration, Utc}; use fancy_regex::Regex; +use log::{error, warn}; use once_cell::sync::Lazy; use rand::Rng; -use reqwest::{header, Client, ClientBuilder, Method, RequestBuilder}; +use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use tokio::sync::Mutex; use crate::{ - cache::Cache, - deobfuscate::Deobfuscator, + cache::{CacheStorage, FileStorage}, + deobfuscate::{DeobfData, Deobfuscator}, model::{Country, Language}, - report::{Level, Report, Reporter, YamlFileReporter}, + report::{JsonFileReporter, Level, Report, Reporter}, + util, }; +/// Client types for accessing the YouTube API. +/// +/// There are multiple clients for accessing the YouTube API which have +/// slightly different features +/// +/// - **Desktop**: used by youtube.com +/// - **DesktopMusic**: used by music.youtube.com, can access special music data, +/// cannot access non-music content +/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos +/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams +/// - **Ios**: used by the iOS app, no obfuscated URLs #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum ClientType { @@ -95,7 +110,7 @@ impl Default for RequestYT { #[derive(Clone, Debug, Serialize, Default)] #[serde(rename_all = "camelCase")] struct User { - // TO DO: provide a way to enable restricted mode with: + // TODO: provide a way to enable restricted mode with: // "enableSafetyMode": true locked_safety_mode: bool, } @@ -131,6 +146,11 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5"; static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]); +/// The RustyPipe client used to access YouTube's API +/// +/// RustyPipe includes an `Arc` internally, so if you are using the client +/// at multiple locations, you can just clone it. Note that options (lang/country/report) +/// are not shared between clones. #[derive(Clone)] pub struct RustyPipe { inner: Arc, @@ -139,10 +159,11 @@ pub struct RustyPipe { struct RustyPipeRef { http: Client, - cache: Cache, + storage: Option>, reporter: Option>, user_agent: String, consent_cookie: String, + cache: Mutex, } #[derive(Clone)] @@ -150,13 +171,14 @@ struct RustyPipeOpts { lang: Language, country: Country, report: bool, + strict: bool, } impl Default for RustyPipe { fn default() -> Self { Self::new( - Some(Cache::from_json_file("RustyPipeCache.json")), - Some(Box::new(YamlFileReporter::default())), + Some(Box::new(FileStorage::default())), + Some(Box::new(JsonFileReporter::default())), None, ) } @@ -168,17 +190,64 @@ impl Default for RustyPipeOpts { lang: Language::En, country: Country::Us, report: false, + strict: false, + } + } +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +struct CacheData { + desktop_client: CacheEntry, + music_client: CacheEntry, + deobf: CacheEntry, +} + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +enum CacheEntry { + #[default] + None, + Some { + last_update: DateTime, + data: T, + }, +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClientData { + pub version: String, +} + +impl CacheEntry { + fn get(&self) -> Option<&T> { + match self { + CacheEntry::Some { last_update, data } => { + if last_update < &(Utc::now() - Duration::hours(24)) { + None + } else { + Some(data) + } + } + CacheEntry::None => None, + } + } +} + +impl From for CacheEntry { + fn from(f: T) -> Self { + Self::Some { + last_update: Utc::now(), + data: f, } } } impl RustyPipe { + /// Create a new RustyPipe instance pub fn new( - cache: Option, + storage: Option>, reporter: Option>, user_agent: Option, ) -> Self { - let cache = cache.unwrap_or_else(|| Cache::default()); let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); let http = ClientBuilder::new() @@ -188,10 +257,26 @@ impl RustyPipe { .build() .expect("unable to build the HTTP client"); + let cache = if let Some(storage) = &storage { + if let Some(data) = storage.read() { + match serde_json::from_str::(&data) { + Ok(data) => data, + Err(e) => { + error!("Could not deserialize cache. Error: {}", e); + CacheData::default() + } + } + } else { + CacheData::default() + } + } else { + CacheData::default() + }; + RustyPipe { inner: Arc::new(RustyPipeRef { http, - cache, + storage, reporter, user_agent, consent_cookie: format!( @@ -200,26 +285,53 @@ impl RustyPipe { CONSENT_COOKIE_YES, rand::thread_rng().gen_range(100..1000) ), + cache: Mutex::new(cache), }), opts: RustyPipeOpts::default(), } } + /// Create a new RustyPipe instance configured for testing + #[cfg(test)] + #[cfg(feature = "yaml")] + pub fn new_test() -> Self { + Self::new( + Some(Box::new(FileStorage::default())), + Some(Box::new(crate::report::YamlFileReporter::default())), + None, + ) + .strict(true) + } + + /// Set the language parameter used when accessing the YouTube API + /// This will change multilanguage video titles, descriptions and textual dates pub fn lang(mut self, lang: Language) -> Self { self.opts.lang = lang; self } + /// Set the country parameter used when accessing the YouTube API. + /// This will change trends and recommended content. pub fn country(mut self, country: Country) -> Self { self.opts.country = country; self } + /// Generate a report on every operation. + /// This should only be used for debugging. pub fn report(mut self, report: bool) -> Self { self.opts.report = report; self } + /// Enable strict mode, causing operations to fail if there + /// are warnings during deserialization (e.g. invalid items). + /// This should only be used for testing. + pub fn strict(mut self, strict: bool) -> Self { + self.opts.strict = strict; + self + } + async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT { let hl = match localized { true => self.opts.lang, @@ -234,7 +346,7 @@ impl RustyPipe { ClientType::Desktop => ContextYT { client: ClientInfo { client_name: "WEB".to_owned(), - client_version: DESKTOP_CLIENT_VERSION.to_owned(), + client_version: self.get_desktop_client_version().await, client_screen: None, device_model: None, platform: "DESKTOP".to_owned(), @@ -249,7 +361,7 @@ impl RustyPipe { ClientType::DesktopMusic => ContextYT { client: ClientInfo { client_name: "WEB_REMIX".to_owned(), - client_version: DESKTOP_MUSIC_CLIENT_VERSION.to_owned(), + client_version: self.get_music_client_version().await, client_screen: None, device_model: None, platform: "DESKTOP".to_owned(), @@ -332,7 +444,7 @@ impl RustyPipe { .header(header::REFERER, "https://www.youtube.com") .header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "1") - .header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION), + .header("X-YouTube-Client-Version", self.get_desktop_client_version().await), ClientType::DesktopMusic => self .inner .http @@ -350,7 +462,7 @@ impl RustyPipe { .header(header::REFERER, "https://music.youtube.com") .header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "67") - .header("X-YouTube-Client-Version", DESKTOP_MUSIC_CLIENT_VERSION), + .header("X-YouTube-Client-Version", self.get_music_client_version().await), ClientType::TvHtml5Embed => self .inner .http @@ -410,7 +522,7 @@ impl RustyPipe { } } - async fn execute_request< + async fn execute_request_deobf< R: DeserializeOwned + MapResponse + Debug, M, B: Serialize + ?Sized, @@ -448,6 +560,7 @@ impl RustyPipe { operation: operation.to_owned(), error, msgs, + deobf_data: deobf.map(Deobfuscator::get_data), http_request: crate::report::HTTPRequest { url: request_url, method: method.to_string(), @@ -482,6 +595,10 @@ impl RustyPipe { Some("Warnings during deserialization/mapping".to_owned()), mapres.warnings, ); + + if self.opts.strict { + bail!("Warnings during deserialization/mapping"); + } } else if self.opts.report { create_report(Level::DBG, None, vec![]); } @@ -500,6 +617,176 @@ impl RustyPipe { } } } + + async fn execute_request< + R: DeserializeOwned + MapResponse + Debug, + M, + B: Serialize + ?Sized, + >( + &self, + ctype: ClientType, + operation: &str, + method: Method, + endpoint: &str, + id: &str, + body: &B, + ) -> Result { + self.execute_request_deobf::(ctype, operation, method, endpoint, id, body, None) + .await + } + + async fn get_desktop_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.desktop_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_desktop_client_version().await { + Ok(version) => { + cache.desktop_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.write_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_CLIENT_VERSION.to_owned() + } + }, + } + } + + async fn get_music_client_version(&self) -> String { + let mut cache = self.inner.cache.lock().await; + + match cache.music_client.get() { + Some(cdata) => cdata.version.to_owned(), + None => match self.extract_music_client_version().await { + Ok(version) => { + cache.music_client = CacheEntry::from(ClientData { + version: version.to_owned(), + }); + self.write_cache(&cache); + version + } + Err(e) => { + warn!("{}, falling back to hardcoded version", e); + DESKTOP_MUSIC_CLIENT_VERSION.to_owned() + } + }, + } + } + + async fn get_deobf(&self) -> Result { + let mut cache = self.inner.cache.lock().await; + let deobf = Deobfuscator::new(self.inner.http.clone()).await?; + cache.deobf = CacheEntry::from(deobf.get_data()); + self.write_cache(&cache); + Ok(deobf) + } + + async fn extract_desktop_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .exec_request_text( + self.inner + .http + .get("https://www.youtube.com/sw.js") + .header(header::ORIGIN, "https://www.youtube.com") + .header(header::REFERER, "https://www.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = self + .exec_request_text( + self.inner + .http + .get("https://www.youtube.com/results?search_query=") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } + } + + async fn extract_music_client_version(&self) -> Result { + let from_swjs = async { + let swjs = self + .exec_request_text( + self.inner + .http + .get("https://music.youtube.com/sw.js") + .header(header::ORIGIN, "https://music.youtube.com") + .header(header::REFERER, "https://music.youtube.com") + .header(header::COOKIE, self.inner.consent_cookie.to_owned()) + .build() + .unwrap(), + ) + .await + .context("Failed to download sw.js")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) + .ok_or(anyhow!("Could not find desktop client version in sw.js")) + }; + + let from_html = async { + let html = self + .exec_request_text( + self.inner + .http + .get("https://music.youtube.com") + .build() + .unwrap(), + ) + .await + .context("Failed to get YT Desktop page")?; + + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!( + "Could not find desktop client version on html page" + )) + }; + + match from_swjs.await { + Ok(client_version) => Ok(client_version), + Err(_) => from_html.await, + } + } + + async fn exec_request(&self, request: Request) -> Result { + Ok(self.inner.http.execute(request).await?.error_for_status()?) + } + + async fn exec_request_text(&self, request: Request) -> Result { + Ok(self.exec_request(request).await?.text().await?) + } + + fn write_cache(&self, cache: &CacheData) { + if let Some(storage) = &self.inner.storage { + match serde_json::to_string(cache) { + Ok(data) => storage.write(&data), + Err(e) => error!("Could not serialize cache. Error: {}", e), + } + } + } } trait MapResponse { @@ -525,10 +812,3 @@ where self.c.fmt(f) } } - -/* -#[cfg(test)] -mod tests { - use super::*; -} -*/ diff --git a/src/client2/player.rs b/src/client2/player.rs index c907389..402b8ba 100644 --- a/src/client2/player.rs +++ b/src/client2/player.rs @@ -59,10 +59,10 @@ struct QContentPlaybackContext { impl RustyPipe { pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result { - let (context, deobf) = tokio::join!( - self.get_context(client_type, false), - Deobfuscator::from_fetched_info(self.inner.http.clone(), self.inner.cache.clone()) - ); + let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf()); + // let context = self.get_context(client_type, false).await; + // let deobf = self.get_deobf().await; + let deobf = deobf?; let request_body = if client_type.is_web() { @@ -90,7 +90,7 @@ impl RustyPipe { } }; - self.execute_request::( + self.execute_request_deobf::( client_type, "get_player", Method::POST, @@ -575,10 +575,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec { } #[cfg(test)] +#[cfg(feature = "yaml")] mod tests { use std::{fs::File, io::BufReader, path::Path}; - use crate::{cache::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; + use crate::{deobfuscate::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; use super::*; use rstest::rstest; @@ -613,7 +614,7 @@ mod tests { #[test_log::test(tokio::test)] async fn download_model_testfiles() { let tf_dir = Path::new("testfiles/player_model"); - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { let mut json_path = tf_dir.to_path_buf(); @@ -683,7 +684,7 @@ mod tests { #[case::ios(ClientType::Ios)] #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap(); // dbg!(&player_data); diff --git a/src/client2/playlist.rs b/src/client2/playlist.rs index e92f1fa..8e3214a 100644 --- a/src/client2/playlist.rs +++ b/src/client2/playlist.rs @@ -40,7 +40,6 @@ impl RustyPipe { "browse", playlist_id, &request_body, - None, ) .await } @@ -62,7 +61,6 @@ impl RustyPipe { "browse", &playlist.id, &request_body, - None, ) .await?; @@ -350,7 +348,7 @@ mod tests { #[case] description: Option, #[case] channel: Option, ) { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let playlist = rp.get_playlist(id).await.unwrap(); assert_eq!(playlist.id, id); @@ -412,7 +410,7 @@ mod tests { #[test_log::test(tokio::test)] async fn t_playlist_cont() { - let rp = RustyPipe::default(); + let rp = RustyPipe::new_test(); let mut playlist = rp .get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi") .await diff --git a/src/deobfuscate.rs b/src/deobfuscate.rs index 0227b37..e85d012 100644 --- a/src/deobfuscate.rs +++ b/src/deobfuscate.rs @@ -3,43 +3,47 @@ use fancy_regex::Regex; use log::debug; use once_cell::sync::Lazy; use reqwest::Client; +use serde::{Serialize, Deserialize}; use std::result::Result::Ok; -use crate::cache::{Cache, DeobfData}; use crate::util; pub struct Deobfuscator { data: DeobfData, } +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DeobfData { + pub js_url: String, + pub sig_fn: String, + pub nsig_fn: String, + pub sts: String, +} + impl Deobfuscator { - pub async fn from_fetched_info(http: Client, cache: Cache) -> Result { - let data = cache - .get_deobf_data(async move { - let js_url = get_player_js_url(&http) - .await - .context("Failed to retrieve player.js URL")?; + pub async fn new(http: Client) -> Result { + let js_url = get_player_js_url(&http) + .await + .context("Failed to retrieve player.js URL")?; - let player_js = get_response(&http, &js_url) - .await - .context("Failed to download player.js")?; + let player_js = get_response(&http, &js_url) + .await + .context("Failed to download player.js")?; - debug!("Downloaded player.js from {}", js_url); + debug!("Downloaded player.js from {}", js_url); - let sig_fn = get_sig_fn(&player_js)?; - let nsig_fn = get_nsig_fn(&player_js)?; - let sts = get_sts(&player_js)?; + let sig_fn = get_sig_fn(&player_js)?; + let nsig_fn = get_nsig_fn(&player_js)?; + let sts = get_sts(&player_js)?; - Ok(DeobfData { - js_url, - nsig_fn, - sig_fn, - sts, - }) - }) - .await?; - - Ok(Self { data }) + Ok(Self { + data: DeobfData { + js_url, + nsig_fn, + sig_fn, + sts, + }, + }) } pub fn deobfuscate_sig(&self, sig: &str) -> Result { @@ -53,6 +57,10 @@ impl Deobfuscator { pub fn get_sts(&self) -> String { self.data.sts.to_owned() } + + pub fn get_data(&self) -> DeobfData { + self.data.to_owned() + } } impl From for Deobfuscator { @@ -472,8 +480,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c #[test(tokio::test)] async fn t_update() { let client = Client::new(); - let cache = Cache::default(); - let deobf = Deobfuscator::from_fetched_info(client, cache) + let deobf = Deobfuscator::new(client) .await .unwrap(); diff --git a/src/report.rs b/src/report.rs index bd5ea16..85939d9 100644 --- a/src/report.rs +++ b/src/report.rs @@ -9,6 +9,8 @@ use chrono::{DateTime, Local}; use log::error; use serde::{Deserialize, Serialize}; +use crate::deobfuscate::DeobfData; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Report { /// Rust package name (`rustypipe`) @@ -25,9 +27,9 @@ pub struct Report { pub error: Option, /// Detailed error/warning messages pub msgs: Vec, - // /// Deobfuscation data (only for player requests) - // #[serde(skip_serializing_if = "Option::is_none")] - // pub deobf_data: Option, + /// Deobfuscation data (only for player requests) + #[serde(skip_serializing_if = "Option::is_none")] + pub deobf_data: Option, /// HTTP request data pub http_request: HTTPRequest, } @@ -96,10 +98,12 @@ impl Reporter for JsonFileReporter { } } +#[cfg(feature="yaml")] pub struct YamlFileReporter { path: PathBuf, } +#[cfg(feature="yaml")] impl YamlFileReporter { pub fn new>(path: P) -> Self { Self { @@ -114,6 +118,7 @@ impl YamlFileReporter { } } +#[cfg(feature="yaml")] impl Default for YamlFileReporter { fn default() -> Self { Self { @@ -122,6 +127,7 @@ impl Default for YamlFileReporter { } } +#[cfg(feature="yaml")] impl Reporter for YamlFileReporter { fn report(&self, report: &Report) { self._report(report)