Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactored cache

Browse source
This commit is contained in:
ThetaDev 2022-09-14 00:04:51 +02:00
parent 925652acdd
commit dda2211e04
8 changed files with 396 additions and 409 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -1,6 +1,5 @@
/target /target
/Cargo.lock /Cargo.lock
RustyPipeReports rustypipe_reports
RustyPipeCache.json rustypipe_cache.json
rusty-tube.json

9
Cargo.toml
View file

@ -7,12 +7,16 @@ edition = "2021"
members = [".", "cli"] members = [".", "cli"]
[features] [features]
default = ["default-tls"] default = ["default-tls", "yaml"]
# Reqwest TLS
default-tls = ["reqwest/default-tls"] default-tls = ["reqwest/default-tls"]
rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"] rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"]
rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
# Error reports in yaml format
yaml = ["serde_yaml"]
[dependencies] [dependencies]
# quick-js = "0.4.1" # quick-js = "0.4.1"
quick-js = { path = "../quickjs-rs" } quick-js = { path = "../quickjs-rs" }
@ -26,10 +30,9 @@ reqwest = {version = "0.11.11", default-features = false, features = ["json", "g
tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} tokio = {version = "1.20.0", features = ["macros", "fs", "process"]}
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.82" serde_json = "1.0.82"
serde_yaml = "0.9.11" serde_yaml = {version = "0.9.11", optional = true}
serde_with = {version = "2.0.0", features = ["json"] } serde_with = {version = "2.0.0", features = ["json"] }
rand = "0.8.5" rand = "0.8.5"
async-trait = "0.1.56"
chrono = {version = "0.4.19", features = ["serde"]} chrono = {version = "0.4.19", features = ["serde"]}
chronoutil = "0.2.3" chronoutil = "0.2.3"
futures = "0.3.21" futures = "0.3.21"

369
src/cache.rs
View file

@ -1,364 +1,57 @@
use std::{ use std::{
fs::File, fs,
future::Future,
io::BufReader,
path::{Path, PathBuf}, path::{Path, PathBuf},
sync::Arc,
}; };
use anyhow::Result; use log::error;
use chrono::{DateTime, Duration, Utc};
use log::{error, info};
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
#[derive(Default, Debug, Clone)] pub trait CacheStorage {
pub struct Cache { fn write(&self, data: &str);
file: Option<PathBuf>, fn read(&self) -> Option<String>;
data: Arc<Mutex<CacheData>>,
} }
#[derive(Default, Debug, Clone, Serialize, Deserialize)] pub struct FileStorage {
struct CacheData { path: PathBuf,
desktop_client: Option<CacheEntry<ClientData>>,
music_client: Option<CacheEntry<ClientData>>,
deobf: Option<CacheEntry<DeobfData>>,
} }
#[derive(Debug, Clone, Serialize, Deserialize)] impl FileStorage {
struct CacheEntry<T> { pub fn new<P: AsRef<Path>>(path: P) -> Self {
last_update: DateTime<Utc>,
data: T,
}
impl<T> From<T> for CacheEntry<T> {
fn from(f: T) -> Self {
Self { Self {
last_update: Utc::now(), path: path.as_ref().to_path_buf(),
data: f,
} }
} }
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] impl Default for FileStorage {
pub struct ClientData { fn default() -> Self {
pub version: String, Self {
path: Path::new("rustypipe_cache.json").into(),
}
}
} }
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] impl CacheStorage for FileStorage {
pub struct DeobfData { fn write(&self, data: &str) {
pub js_url: String, fs::write(&self.path, data).unwrap_or_else(|e| {
pub sig_fn: String, error!(
pub nsig_fn: String, "Could not write cache to file `{}`. Error: {}",
pub sts: String, self.path.to_string_lossy(),
} e
);
impl Cache { });
pub async fn get_desktop_client_data<F>(&self, updater: F) -> Result<ClientData>
where
F: Future<Output = Result<ClientData>> + Send + 'static,
{
let mut cache = self.data.lock().await;
if cache.desktop_client.is_none()
|| cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
{
let cdata = updater.await?;
cache.desktop_client = Some(CacheEntry::from(cdata.clone()));
self.save(&cache);
Ok(cdata)
} else {
Ok(cache.desktop_client.as_ref().unwrap().data.clone())
}
} }
pub async fn get_music_client_data<F>(&self, updater: F) -> Result<ClientData> fn read(&self) -> Option<String> {
where match fs::read_to_string(&self.path) {
F: Future<Output = Result<ClientData>> + Send + 'static, Ok(data) => Some(data),
{
let mut cache = self.data.lock().await;
if cache.music_client.is_none()
|| cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
{
let cdata = updater.await?;
cache.music_client = Some(CacheEntry::from(cdata.clone()));
self.save(&cache);
Ok(cdata)
} else {
Ok(cache.music_client.as_ref().unwrap().data.clone())
}
}
pub async fn get_deobf_data<F>(&self, updater: F) -> Result<DeobfData>
where
F: Future<Output = Result<DeobfData>> + Send + 'static,
{
let mut cache = self.data.lock().await;
if cache.deobf.is_none()
|| cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
{
let deobf_data = updater.await?;
cache.deobf = Some(CacheEntry::from(deobf_data.clone()));
self.save(&cache);
Ok(deobf_data)
} else {
Ok(cache.deobf.as_ref().unwrap().data.clone())
}
}
pub async fn to_json(&self) -> Result<String> {
let cache = self.data.lock().await;
Ok(serde_json::to_string(&cache.clone())?)
}
pub async fn to_json_file<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let cache = self.data.lock().await;
Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?)
}
pub fn from_json(json: &str) -> Self {
let data: CacheData = match serde_json::from_str(json) {
Ok(cd) => cd,
Err(e) => { Err(e) => {
error!( error!(
"Could not load cache from json, falling back to default. Error: {}", "Could not load cache from file `{}`. Error: {}",
self.path.to_string_lossy(),
e e
); );
CacheData::default() None
} }
};
Cache {
data: Arc::new(Mutex::new(data)),
file: None,
}
}
pub fn from_json_file<P: AsRef<Path>>(path: P) -> Self {
let file = match File::open(path.as_ref()) {
Ok(file) => file,
Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound {
info!(
"Cache json file at {} not found, will be created",
path.as_ref().to_string_lossy()
)
} else {
error!(
"Could not open cache json file, falling back to default. Error: {}",
e
);
}
return Cache {
file: Some(path.as_ref().to_path_buf()),
..Default::default()
};
}
};
let data: CacheData = match serde_json::from_reader(BufReader::new(file)) {
Ok(data) => data,
Err(e) => {
error!(
"Could not load cache from json, falling back to default. Error: {}",
e
);
return Cache {
file: Some(path.as_ref().to_path_buf()),
..Default::default()
};
}
};
Cache {
data: Arc::new(Mutex::new(data)),
file: Some(path.as_ref().to_path_buf()),
}
}
fn save(&self, cache: &CacheData) {
match self.file.as_ref() {
Some(file) => match File::create(file) {
Ok(file) => match serde_json::to_writer(file, cache) {
Ok(_) => {}
Err(e) => error!("Could not write cache to json. Error: {}", e),
},
Err(e) => error!("Could not open cache json file. Error: {}", e),
},
None => {}
} }
} }
} }
#[cfg(test)]
mod tests {
use temp_testdir::TempDir;
use super::*;
#[tokio::test]
async fn test() {
let cache = Cache::default();
let desktop_c = cache
.get_desktop_client_data(async {
Ok(ClientData {
version: "1.2.3".to_owned(),
})
})
.await
.unwrap();
assert_eq!(
desktop_c,
ClientData {
version: "1.2.3".to_owned()
}
);
let music_c = cache
.get_music_client_data(async {
Ok(ClientData {
version: "4.5.6".to_owned(),
})
})
.await
.unwrap();
assert_eq!(
music_c,
ClientData {
version: "4.5.6".to_owned()
}
);
let deobf_data = cache
.get_deobf_data(async {
Ok(DeobfData {
js_url:
"https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
.to_owned(),
sig_fn: "t_sig_fn".to_owned(),
nsig_fn: "t_nsig_fn".to_owned(),
sts: "t_sts".to_owned(),
})
})
.await
.unwrap();
assert_eq!(
deobf_data,
DeobfData {
js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
.to_owned(),
sig_fn: "t_sig_fn".to_owned(),
nsig_fn: "t_nsig_fn".to_owned(),
sts: "t_sts".to_owned(),
}
);
// Create a new cache from the first one's json
// and check if it returns the same cached data
let json = cache.to_json().await.unwrap();
let new_cache = Cache::from_json(&json);
assert_eq!(
new_cache
.get_desktop_client_data(async {
Ok(ClientData {
version: "".to_owned(),
})
})
.await
.unwrap(),
desktop_c
);
assert_eq!(
new_cache
.get_music_client_data(async {
Ok(ClientData {
version: "".to_owned(),
})
})
.await
.unwrap(),
music_c
);
assert_eq!(
new_cache
.get_deobf_data(async {
Ok(DeobfData {
js_url: "".to_owned(),
nsig_fn: "".to_owned(),
sig_fn: "".to_owned(),
sts: "".to_owned(),
})
})
.await
.unwrap(),
deobf_data
);
}
#[tokio::test]
async fn test_file() {
let temp = TempDir::default();
let mut file_path = PathBuf::from(temp.as_ref());
file_path.push("cache.json");
let cache = Cache::from_json_file(file_path.clone());
let cdata = cache
.get_desktop_client_data(async {
Ok(ClientData {
version: "1.2.3".to_owned(),
})
})
.await
.unwrap();
let deobf_data = cache
.get_deobf_data(async {
Ok(DeobfData {
js_url:
"https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
.to_owned(),
sig_fn: "t_sig_fn".to_owned(),
nsig_fn: "t_nsig_fn".to_owned(),
sts: "t_sts".to_owned(),
})
})
.await
.unwrap();
assert!(file_path.exists());
let new_cache = Cache::from_json_file(file_path.clone());
assert_eq!(
new_cache
.get_desktop_client_data(async {
Ok(ClientData {
version: "".to_owned(),
})
})
.await
.unwrap(),
cdata
);
assert_eq!(
new_cache
.get_deobf_data(async {
Ok(DeobfData {
js_url: "".to_owned(),
nsig_fn: "".to_owned(),
sig_fn: "".to_owned(),
sts: "".to_owned(),
})
})
.await
.unwrap(),
deobf_data
);
}
}

328
src/client2/mod.rs
View file

@ -6,20 +6,35 @@ mod response;
use std::fmt::Debug; use std::fmt::Debug;
use std::sync::Arc; use std::sync::Arc;
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, bail, Context, Result};
use chrono::{DateTime, Duration, Utc};
use fancy_regex::Regex; use fancy_regex::Regex;
use log::{error, warn};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use rand::Rng; use rand::Rng;
use reqwest::{header, Client, ClientBuilder, Method, RequestBuilder}; use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response};
use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde::{de::DeserializeOwned, Deserialize, Serialize};
use tokio::sync::Mutex;
use crate::{ use crate::{
cache::Cache, cache::{CacheStorage, FileStorage},
deobfuscate::Deobfuscator, deobfuscate::{DeobfData, Deobfuscator},
model::{Country, Language}, model::{Country, Language},
report::{Level, Report, Reporter, YamlFileReporter}, report::{JsonFileReporter, Level, Report, Reporter},
util,
}; };
/// Client types for accessing the YouTube API.
///
/// There are multiple clients for accessing the YouTube API which have
/// slightly different features
///
/// - **Desktop**: used by youtube.com
/// - **DesktopMusic**: used by music.youtube.com, can access special music data,
/// cannot access non-music content
/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos
/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams
/// - **Ios**: used by the iOS app, no obfuscated URLs
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")] #[serde(rename_all = "snake_case")]
pub enum ClientType { pub enum ClientType {
@ -95,7 +110,7 @@ impl Default for RequestYT {
#[derive(Clone, Debug, Serialize, Default)] #[derive(Clone, Debug, Serialize, Default)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct User { struct User {
// TO DO: provide a way to enable restricted mode with: // TODO: provide a way to enable restricted mode with:
// "enableSafetyMode": true // "enableSafetyMode": true
locked_safety_mode: bool, locked_safety_mode: bool,
} }
@ -131,6 +146,11 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5";
static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> = static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> =
Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]); Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]);
/// The RustyPipe client used to access YouTube's API
///
/// RustyPipe includes an `Arc` internally, so if you are using the client
/// at multiple locations, you can just clone it. Note that options (lang/country/report)
/// are not shared between clones.
#[derive(Clone)] #[derive(Clone)]
pub struct RustyPipe { pub struct RustyPipe {
inner: Arc<RustyPipeRef>, inner: Arc<RustyPipeRef>,
@ -139,10 +159,11 @@ pub struct RustyPipe {
struct RustyPipeRef { struct RustyPipeRef {
http: Client, http: Client,
cache: Cache, storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>, reporter: Option<Box<dyn Reporter>>,
user_agent: String, user_agent: String,
consent_cookie: String, consent_cookie: String,
cache: Mutex<CacheData>,
} }
#[derive(Clone)] #[derive(Clone)]
@ -150,13 +171,14 @@ struct RustyPipeOpts {
lang: Language, lang: Language,
country: Country, country: Country,
report: bool, report: bool,
strict: bool,
} }
impl Default for RustyPipe { impl Default for RustyPipe {
fn default() -> Self { fn default() -> Self {
Self::new( Self::new(
Some(Cache::from_json_file("RustyPipeCache.json")), Some(Box::new(FileStorage::default())),
Some(Box::new(YamlFileReporter::default())), Some(Box::new(JsonFileReporter::default())),
None, None,
) )
} }
@ -168,17 +190,64 @@ impl Default for RustyPipeOpts {
lang: Language::En, lang: Language::En,
country: Country::Us, country: Country::Us,
report: false, report: false,
strict: false,
}
}
}
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
struct CacheData {
desktop_client: CacheEntry<ClientData>,
music_client: CacheEntry<ClientData>,
deobf: CacheEntry<DeobfData>,
}
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
enum CacheEntry<T> {
#[default]
None,
Some {
last_update: DateTime<Utc>,
data: T,
},
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ClientData {
pub version: String,
}
impl<T> CacheEntry<T> {
fn get(&self) -> Option<&T> {
match self {
CacheEntry::Some { last_update, data } => {
if last_update < &(Utc::now() - Duration::hours(24)) {
None
} else {
Some(data)
}
}
CacheEntry::None => None,
}
}
}
impl<T> From<T> for CacheEntry<T> {
fn from(f: T) -> Self {
Self::Some {
last_update: Utc::now(),
data: f,
} }
} }
} }
impl RustyPipe { impl RustyPipe {
/// Create a new RustyPipe instance
pub fn new( pub fn new(
cache: Option<Cache>, storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>, reporter: Option<Box<dyn Reporter>>,
user_agent: Option<String>, user_agent: Option<String>,
) -> Self { ) -> Self {
let cache = cache.unwrap_or_else(|| Cache::default());
let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned());
let http = ClientBuilder::new() let http = ClientBuilder::new()
@ -188,10 +257,26 @@ impl RustyPipe {
.build() .build()
.expect("unable to build the HTTP client"); .expect("unable to build the HTTP client");
let cache = if let Some(storage) = &storage {
if let Some(data) = storage.read() {
match serde_json::from_str::<CacheData>(&data) {
Ok(data) => data,
Err(e) => {
error!("Could not deserialize cache. Error: {}", e);
CacheData::default()
}
}
} else {
CacheData::default()
}
} else {
CacheData::default()
};
RustyPipe { RustyPipe {
inner: Arc::new(RustyPipeRef { inner: Arc::new(RustyPipeRef {
http, http,
cache, storage,
reporter, reporter,
user_agent, user_agent,
consent_cookie: format!( consent_cookie: format!(
@ -200,26 +285,53 @@ impl RustyPipe {
CONSENT_COOKIE_YES, CONSENT_COOKIE_YES,
rand::thread_rng().gen_range(100..1000) rand::thread_rng().gen_range(100..1000)
), ),
cache: Mutex::new(cache),
}), }),
opts: RustyPipeOpts::default(), opts: RustyPipeOpts::default(),
} }
} }
/// Create a new RustyPipe instance configured for testing
#[cfg(test)]
#[cfg(feature = "yaml")]
pub fn new_test() -> Self {
Self::new(
Some(Box::new(FileStorage::default())),
Some(Box::new(crate::report::YamlFileReporter::default())),
None,
)
.strict(true)
}
/// Set the language parameter used when accessing the YouTube API
/// This will change multilanguage video titles, descriptions and textual dates
pub fn lang(mut self, lang: Language) -> Self { pub fn lang(mut self, lang: Language) -> Self {
self.opts.lang = lang; self.opts.lang = lang;
self self
} }
/// Set the country parameter used when accessing the YouTube API.
/// This will change trends and recommended content.
pub fn country(mut self, country: Country) -> Self { pub fn country(mut self, country: Country) -> Self {
self.opts.country = country; self.opts.country = country;
self self
} }
/// Generate a report on every operation.
/// This should only be used for debugging.
pub fn report(mut self, report: bool) -> Self { pub fn report(mut self, report: bool) -> Self {
self.opts.report = report; self.opts.report = report;
self self
} }
/// Enable strict mode, causing operations to fail if there
/// are warnings during deserialization (e.g. invalid items).
/// This should only be used for testing.
pub fn strict(mut self, strict: bool) -> Self {
self.opts.strict = strict;
self
}
async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT { async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT {
let hl = match localized { let hl = match localized {
true => self.opts.lang, true => self.opts.lang,
@ -234,7 +346,7 @@ impl RustyPipe {
ClientType::Desktop => ContextYT { ClientType::Desktop => ContextYT {
client: ClientInfo { client: ClientInfo {
client_name: "WEB".to_owned(), client_name: "WEB".to_owned(),
client_version: DESKTOP_CLIENT_VERSION.to_owned(), client_version: self.get_desktop_client_version().await,
client_screen: None, client_screen: None,
device_model: None, device_model: None,
platform: "DESKTOP".to_owned(), platform: "DESKTOP".to_owned(),
@ -249,7 +361,7 @@ impl RustyPipe {
ClientType::DesktopMusic => ContextYT { ClientType::DesktopMusic => ContextYT {
client: ClientInfo { client: ClientInfo {
client_name: "WEB_REMIX".to_owned(), client_name: "WEB_REMIX".to_owned(),
client_version: DESKTOP_MUSIC_CLIENT_VERSION.to_owned(), client_version: self.get_music_client_version().await,
client_screen: None, client_screen: None,
device_model: None, device_model: None,
platform: "DESKTOP".to_owned(), platform: "DESKTOP".to_owned(),
@ -332,7 +444,7 @@ impl RustyPipe {
.header(header::REFERER, "https://www.youtube.com") .header(header::REFERER, "https://www.youtube.com")
.header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header(header::COOKIE, self.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "1") .header("X-YouTube-Client-Name", "1")
.header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION), .header("X-YouTube-Client-Version", self.get_desktop_client_version().await),
ClientType::DesktopMusic => self ClientType::DesktopMusic => self
.inner .inner
.http .http
@ -350,7 +462,7 @@ impl RustyPipe {
.header(header::REFERER, "https://music.youtube.com") .header(header::REFERER, "https://music.youtube.com")
.header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header(header::COOKIE, self.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "67") .header("X-YouTube-Client-Name", "67")
.header("X-YouTube-Client-Version", DESKTOP_MUSIC_CLIENT_VERSION), .header("X-YouTube-Client-Version", self.get_music_client_version().await),
ClientType::TvHtml5Embed => self ClientType::TvHtml5Embed => self
.inner .inner
.http .http
@ -410,7 +522,7 @@ impl RustyPipe {
} }
} }
async fn execute_request< async fn execute_request_deobf<
R: DeserializeOwned + MapResponse<M> + Debug, R: DeserializeOwned + MapResponse<M> + Debug,
M, M,
B: Serialize + ?Sized, B: Serialize + ?Sized,
@ -448,6 +560,7 @@ impl RustyPipe {
operation: operation.to_owned(), operation: operation.to_owned(),
error, error,
msgs, msgs,
deobf_data: deobf.map(Deobfuscator::get_data),
http_request: crate::report::HTTPRequest { http_request: crate::report::HTTPRequest {
url: request_url, url: request_url,
method: method.to_string(), method: method.to_string(),
@ -482,6 +595,10 @@ impl RustyPipe {
Some("Warnings during deserialization/mapping".to_owned()), Some("Warnings during deserialization/mapping".to_owned()),
mapres.warnings, mapres.warnings,
); );
if self.opts.strict {
bail!("Warnings during deserialization/mapping");
}
} else if self.opts.report { } else if self.opts.report {
create_report(Level::DBG, None, vec![]); create_report(Level::DBG, None, vec![]);
} }
@ -500,6 +617,176 @@ impl RustyPipe {
} }
} }
} }
async fn execute_request<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
B: Serialize + ?Sized,
>(
&self,
ctype: ClientType,
operation: &str,
method: Method,
endpoint: &str,
id: &str,
body: &B,
) -> Result<M> {
self.execute_request_deobf::<R, M, B>(ctype, operation, method, endpoint, id, body, None)
.await
}
async fn get_desktop_client_version(&self) -> String {
let mut cache = self.inner.cache.lock().await;
match cache.desktop_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => match self.extract_desktop_client_version().await {
Ok(version) => {
cache.desktop_client = CacheEntry::from(ClientData {
version: version.to_owned(),
});
self.write_cache(&cache);
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
DESKTOP_CLIENT_VERSION.to_owned()
}
},
}
}
async fn get_music_client_version(&self) -> String {
let mut cache = self.inner.cache.lock().await;
match cache.music_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => match self.extract_music_client_version().await {
Ok(version) => {
cache.music_client = CacheEntry::from(ClientData {
version: version.to_owned(),
});
self.write_cache(&cache);
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
}
},
}
}
async fn get_deobf(&self) -> Result<Deobfuscator> {
let mut cache = self.inner.cache.lock().await;
let deobf = Deobfuscator::new(self.inner.http.clone()).await?;
cache.deobf = CacheEntry::from(deobf.get_data());
self.write_cache(&cache);
Ok(deobf)
}
async fn extract_desktop_client_version(&self) -> Result<String> {
let from_swjs = async {
let swjs = self
.exec_request_text(
self.inner
.http
.get("https://www.youtube.com/sw.js")
.header(header::ORIGIN, "https://www.youtube.com")
.header(header::REFERER, "https://www.youtube.com")
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
.build()
.unwrap(),
)
.await
.context("Failed to download sw.js")?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
.ok_or(anyhow!("Could not find desktop client version in sw.js"))
};
let from_html = async {
let html = self
.exec_request_text(
self.inner
.http
.get("https://www.youtube.com/results?search_query=")
.build()
.unwrap(),
)
.await
.context("Failed to get YT Desktop page")?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!(
"Could not find desktop client version on html page"
))
};
match from_swjs.await {
Ok(client_version) => Ok(client_version),
Err(_) => from_html.await,
}
}
async fn extract_music_client_version(&self) -> Result<String> {
let from_swjs = async {
let swjs = self
.exec_request_text(
self.inner
.http
.get("https://music.youtube.com/sw.js")
.header(header::ORIGIN, "https://music.youtube.com")
.header(header::REFERER, "https://music.youtube.com")
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
.build()
.unwrap(),
)
.await
.context("Failed to download sw.js")?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
.ok_or(anyhow!("Could not find desktop client version in sw.js"))
};
let from_html = async {
let html = self
.exec_request_text(
self.inner
.http
.get("https://music.youtube.com")
.build()
.unwrap(),
)
.await
.context("Failed to get YT Desktop page")?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!(
"Could not find desktop client version on html page"
))
};
match from_swjs.await {
Ok(client_version) => Ok(client_version),
Err(_) => from_html.await,
}
}
async fn exec_request(&self, request: Request) -> Result<Response> {
Ok(self.inner.http.execute(request).await?.error_for_status()?)
}
async fn exec_request_text(&self, request: Request) -> Result<String> {
Ok(self.exec_request(request).await?.text().await?)
}
fn write_cache(&self, cache: &CacheData) {
if let Some(storage) = &self.inner.storage {
match serde_json::to_string(cache) {
Ok(data) => storage.write(&data),
Err(e) => error!("Could not serialize cache. Error: {}", e),
}
}
}
} }
trait MapResponse<T> { trait MapResponse<T> {
@ -525,10 +812,3 @@ where
self.c.fmt(f) self.c.fmt(f)
} }
} }
/*
#[cfg(test)]
mod tests {
use super::*;
}
*/

17
src/client2/player.rs
View file

@ -59,10 +59,10 @@ struct QContentPlaybackContext {
impl RustyPipe { impl RustyPipe {
pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> { pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
let (context, deobf) = tokio::join!( let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf());
self.get_context(client_type, false), // let context = self.get_context(client_type, false).await;
Deobfuscator::from_fetched_info(self.inner.http.clone(), self.inner.cache.clone()) // let deobf = self.get_deobf().await;
);
let deobf = deobf?; let deobf = deobf?;
let request_body = if client_type.is_web() { let request_body = if client_type.is_web() {
@ -90,7 +90,7 @@ impl RustyPipe {
} }
}; };
self.execute_request::<response::Player, _, _>( self.execute_request_deobf::<response::Player, _, _>(
client_type, client_type,
"get_player", "get_player",
Method::POST, Method::POST,
@ -575,10 +575,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec {
} }
#[cfg(test)] #[cfg(test)]
#[cfg(feature = "yaml")]
mod tests { mod tests {
use std::{fs::File, io::BufReader, path::Path}; use std::{fs::File, io::BufReader, path::Path};
use crate::{cache::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter}; use crate::{deobfuscate::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter};
use super::*; use super::*;
use rstest::rstest; use rstest::rstest;
@ -613,7 +614,7 @@ mod tests {
#[test_log::test(tokio::test)] #[test_log::test(tokio::test)]
async fn download_model_testfiles() { async fn download_model_testfiles() {
let tf_dir = Path::new("testfiles/player_model"); let tf_dir = Path::new("testfiles/player_model");
let rp = RustyPipe::default(); let rp = RustyPipe::new_test();
for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] { for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] {
let mut json_path = tf_dir.to_path_buf(); let mut json_path = tf_dir.to_path_buf();
@ -683,7 +684,7 @@ mod tests {
#[case::ios(ClientType::Ios)] #[case::ios(ClientType::Ios)]
#[test_log::test(tokio::test)] #[test_log::test(tokio::test)]
async fn t_get_player(#[case] client_type: ClientType) { async fn t_get_player(#[case] client_type: ClientType) {
let rp = RustyPipe::default(); let rp = RustyPipe::new_test();
let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap(); let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap();
// dbg!(&player_data); // dbg!(&player_data);

6
src/client2/playlist.rs
View file

@ -40,7 +40,6 @@ impl RustyPipe {
"browse", "browse",
playlist_id, playlist_id,
&request_body, &request_body,
None,
) )
.await .await
} }
@ -62,7 +61,6 @@ impl RustyPipe {
"browse", "browse",
&playlist.id, &playlist.id,
&request_body, &request_body,
None,
) )
.await?; .await?;
@ -350,7 +348,7 @@ mod tests {
#[case] description: Option<String>, #[case] description: Option<String>,
#[case] channel: Option<Channel>, #[case] channel: Option<Channel>,
) { ) {
let rp = RustyPipe::default(); let rp = RustyPipe::new_test();
let playlist = rp.get_playlist(id).await.unwrap(); let playlist = rp.get_playlist(id).await.unwrap();
assert_eq!(playlist.id, id); assert_eq!(playlist.id, id);
@ -412,7 +410,7 @@ mod tests {
#[test_log::test(tokio::test)] #[test_log::test(tokio::test)]
async fn t_playlist_cont() { async fn t_playlist_cont() {
let rp = RustyPipe::default(); let rp = RustyPipe::new_test();
let mut playlist = rp let mut playlist = rp
.get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi") .get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")
.await .await

59
src/deobfuscate.rs
View file

@ -3,43 +3,47 @@ use fancy_regex::Regex;
use log::debug; use log::debug;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use reqwest::Client; use reqwest::Client;
use serde::{Serialize, Deserialize};
use std::result::Result::Ok; use std::result::Result::Ok;
use crate::cache::{Cache, DeobfData};
use crate::util; use crate::util;
pub struct Deobfuscator { pub struct Deobfuscator {
data: DeobfData, data: DeobfData,
} }
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DeobfData {
pub js_url: String,
pub sig_fn: String,
pub nsig_fn: String,
pub sts: String,
}
impl Deobfuscator { impl Deobfuscator {
pub async fn from_fetched_info(http: Client, cache: Cache) -> Result<Self> { pub async fn new(http: Client) -> Result<Self> {
let data = cache let js_url = get_player_js_url(&http)
.get_deobf_data(async move { .await
let js_url = get_player_js_url(&http) .context("Failed to retrieve player.js URL")?;
.await
.context("Failed to retrieve player.js URL")?;
let player_js = get_response(&http, &js_url) let player_js = get_response(&http, &js_url)
.await .await
.context("Failed to download player.js")?; .context("Failed to download player.js")?;
debug!("Downloaded player.js from {}", js_url); debug!("Downloaded player.js from {}", js_url);
let sig_fn = get_sig_fn(&player_js)?; let sig_fn = get_sig_fn(&player_js)?;
let nsig_fn = get_nsig_fn(&player_js)?; let nsig_fn = get_nsig_fn(&player_js)?;
let sts = get_sts(&player_js)?; let sts = get_sts(&player_js)?;
Ok(DeobfData { Ok(Self {
js_url, data: DeobfData {
nsig_fn, js_url,
sig_fn, nsig_fn,
sts, sig_fn,
}) sts,
}) },
.await?; })
Ok(Self { data })
} }
pub fn deobfuscate_sig(&self, sig: &str) -> Result<String> { pub fn deobfuscate_sig(&self, sig: &str) -> Result<String> {
@ -53,6 +57,10 @@ impl Deobfuscator {
pub fn get_sts(&self) -> String { pub fn get_sts(&self) -> String {
self.data.sts.to_owned() self.data.sts.to_owned()
} }
pub fn get_data(&self) -> DeobfData {
self.data.to_owned()
}
} }
impl From<DeobfData> for Deobfuscator { impl From<DeobfData> for Deobfuscator {
@ -472,8 +480,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
#[test(tokio::test)] #[test(tokio::test)]
async fn t_update() { async fn t_update() {
let client = Client::new(); let client = Client::new();
let cache = Cache::default(); let deobf = Deobfuscator::new(client)
let deobf = Deobfuscator::from_fetched_info(client, cache)
.await .await
.unwrap(); .unwrap();

12
src/report.rs
View file

@ -9,6 +9,8 @@ use chrono::{DateTime, Local};
use log::error; use log::error;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::deobfuscate::DeobfData;
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Report { pub struct Report {
/// Rust package name (`rustypipe`) /// Rust package name (`rustypipe`)
@ -25,9 +27,9 @@ pub struct Report {
pub error: Option<String>, pub error: Option<String>,
/// Detailed error/warning messages /// Detailed error/warning messages
pub msgs: Vec<String>, pub msgs: Vec<String>,
// /// Deobfuscation data (only for player requests) /// Deobfuscation data (only for player requests)
// #[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
// pub deobf_data: Option<DeobfData>, pub deobf_data: Option<DeobfData>,
/// HTTP request data /// HTTP request data
pub http_request: HTTPRequest, pub http_request: HTTPRequest,
} }
@ -96,10 +98,12 @@ impl Reporter for JsonFileReporter {
} }
} }
#[cfg(feature="yaml")]
pub struct YamlFileReporter { pub struct YamlFileReporter {
path: PathBuf, path: PathBuf,
} }
#[cfg(feature="yaml")]
impl YamlFileReporter { impl YamlFileReporter {
pub fn new<P: AsRef<Path>>(path: P) -> Self { pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self { Self {
@ -114,6 +118,7 @@ impl YamlFileReporter {
} }
} }
#[cfg(feature="yaml")]
impl Default for YamlFileReporter { impl Default for YamlFileReporter {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -122,6 +127,7 @@ impl Default for YamlFileReporter {
} }
} }
#[cfg(feature="yaml")]
impl Reporter for YamlFileReporter { impl Reporter for YamlFileReporter {
fn report(&self, report: &Report) { fn report(&self, report: &Report) {
self._report(report) self._report(report)