refactored cache
This commit is contained in:
ThetaDev
parent
925652acdd
commit
dda2211e04
8 changed files with 396 additions and 409 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -1,6 +1,5 @@
|
|||
/target
|
||||
/Cargo.lock
|
||||
|
||||
RustyPipeReports
|
||||
RustyPipeCache.json
|
||||
rusty-tube.json
|
||||
rustypipe_reports
|
||||
rustypipe_cache.json
|
||||
|
|
|
|||
|
|
@ -7,12 +7,16 @@ edition = "2021"
|
|||
members = [".", "cli"]
|
||||
|
||||
[features]
|
||||
default = ["default-tls"]
|
||||
default = ["default-tls", "yaml"]
|
||||
|
||||
# Reqwest TLS
|
||||
default-tls = ["reqwest/default-tls"]
|
||||
rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"]
|
||||
rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
|
||||
|
||||
# Error reports in yaml format
|
||||
yaml = ["serde_yaml"]
|
||||
|
||||
[dependencies]
|
||||
# quick-js = "0.4.1"
|
||||
quick-js = { path = "../quickjs-rs" }
|
||||
|
|
@ -26,10 +30,9 @@ reqwest = {version = "0.11.11", default-features = false, features = ["json", "g
|
|||
tokio = {version = "1.20.0", features = ["macros", "fs", "process"]}
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0.82"
|
||||
serde_yaml = "0.9.11"
|
||||
serde_yaml = {version = "0.9.11", optional = true}
|
||||
serde_with = {version = "2.0.0", features = ["json"] }
|
||||
rand = "0.8.5"
|
||||
async-trait = "0.1.56"
|
||||
chrono = {version = "0.4.19", features = ["serde"]}
|
||||
chronoutil = "0.2.3"
|
||||
futures = "0.3.21"
|
||||
|
|
|
|||
369
src/cache.rs
369
src/cache.rs
|
|
@ -1,364 +1,57 @@
|
|||
use std::{
|
||||
fs::File,
|
||||
future::Future,
|
||||
io::BufReader,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use anyhow::Result;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use log::{error, info};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::Mutex;
|
||||
use log::error;
|
||||
|
||||
#[derive(Default, Debug, Clone)]
|
||||
pub struct Cache {
|
||||
file: Option<PathBuf>,
|
||||
data: Arc<Mutex<CacheData>>,
|
||||
pub trait CacheStorage {
|
||||
fn write(&self, data: &str);
|
||||
fn read(&self) -> Option<String>;
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
|
||||
struct CacheData {
|
||||
desktop_client: Option<CacheEntry<ClientData>>,
|
||||
music_client: Option<CacheEntry<ClientData>>,
|
||||
deobf: Option<CacheEntry<DeobfData>>,
|
||||
pub struct FileStorage {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct CacheEntry<T> {
|
||||
last_update: DateTime<Utc>,
|
||||
data: T,
|
||||
}
|
||||
|
||||
impl<T> From<T> for CacheEntry<T> {
|
||||
fn from(f: T) -> Self {
|
||||
impl FileStorage {
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
||||
Self {
|
||||
last_update: Utc::now(),
|
||||
data: f,
|
||||
path: path.as_ref().to_path_buf(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ClientData {
|
||||
pub version: String,
|
||||
impl Default for FileStorage {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
path: Path::new("rustypipe_cache.json").into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DeobfData {
|
||||
pub js_url: String,
|
||||
pub sig_fn: String,
|
||||
pub nsig_fn: String,
|
||||
pub sts: String,
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub async fn get_desktop_client_data<F>(&self, updater: F) -> Result<ClientData>
|
||||
where
|
||||
F: Future<Output = Result<ClientData>> + Send + 'static,
|
||||
{
|
||||
let mut cache = self.data.lock().await;
|
||||
|
||||
if cache.desktop_client.is_none()
|
||||
|| cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
|
||||
{
|
||||
let cdata = updater.await?;
|
||||
cache.desktop_client = Some(CacheEntry::from(cdata.clone()));
|
||||
self.save(&cache);
|
||||
Ok(cdata)
|
||||
} else {
|
||||
Ok(cache.desktop_client.as_ref().unwrap().data.clone())
|
||||
}
|
||||
impl CacheStorage for FileStorage {
|
||||
fn write(&self, data: &str) {
|
||||
fs::write(&self.path, data).unwrap_or_else(|e| {
|
||||
error!(
|
||||
"Could not write cache to file `{}`. Error: {}",
|
||||
self.path.to_string_lossy(),
|
||||
e
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn get_music_client_data<F>(&self, updater: F) -> Result<ClientData>
|
||||
where
|
||||
F: Future<Output = Result<ClientData>> + Send + 'static,
|
||||
{
|
||||
let mut cache = self.data.lock().await;
|
||||
|
||||
if cache.music_client.is_none()
|
||||
|| cache.music_client.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
|
||||
{
|
||||
let cdata = updater.await?;
|
||||
cache.music_client = Some(CacheEntry::from(cdata.clone()));
|
||||
self.save(&cache);
|
||||
Ok(cdata)
|
||||
} else {
|
||||
Ok(cache.music_client.as_ref().unwrap().data.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_deobf_data<F>(&self, updater: F) -> Result<DeobfData>
|
||||
where
|
||||
F: Future<Output = Result<DeobfData>> + Send + 'static,
|
||||
{
|
||||
let mut cache = self.data.lock().await;
|
||||
if cache.deobf.is_none()
|
||||
|| cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::hours(24)
|
||||
{
|
||||
let deobf_data = updater.await?;
|
||||
cache.deobf = Some(CacheEntry::from(deobf_data.clone()));
|
||||
self.save(&cache);
|
||||
Ok(deobf_data)
|
||||
} else {
|
||||
Ok(cache.deobf.as_ref().unwrap().data.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn to_json(&self) -> Result<String> {
|
||||
let cache = self.data.lock().await;
|
||||
Ok(serde_json::to_string(&cache.clone())?)
|
||||
}
|
||||
|
||||
pub async fn to_json_file<P: AsRef<Path>>(&self, path: P) -> Result<()> {
|
||||
let cache = self.data.lock().await;
|
||||
Ok(serde_json::to_writer(&File::create(path)?, &cache.clone())?)
|
||||
}
|
||||
|
||||
pub fn from_json(json: &str) -> Self {
|
||||
let data: CacheData = match serde_json::from_str(json) {
|
||||
Ok(cd) => cd,
|
||||
fn read(&self) -> Option<String> {
|
||||
match fs::read_to_string(&self.path) {
|
||||
Ok(data) => Some(data),
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Could not load cache from json, falling back to default. Error: {}",
|
||||
"Could not load cache from file `{}`. Error: {}",
|
||||
self.path.to_string_lossy(),
|
||||
e
|
||||
);
|
||||
CacheData::default()
|
||||
None
|
||||
}
|
||||
};
|
||||
Cache {
|
||||
data: Arc::new(Mutex::new(data)),
|
||||
file: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_json_file<P: AsRef<Path>>(path: P) -> Self {
|
||||
let file = match File::open(path.as_ref()) {
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
if e.kind() == std::io::ErrorKind::NotFound {
|
||||
info!(
|
||||
"Cache json file at {} not found, will be created",
|
||||
path.as_ref().to_string_lossy()
|
||||
)
|
||||
} else {
|
||||
error!(
|
||||
"Could not open cache json file, falling back to default. Error: {}",
|
||||
e
|
||||
);
|
||||
}
|
||||
return Cache {
|
||||
file: Some(path.as_ref().to_path_buf()),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
};
|
||||
let data: CacheData = match serde_json::from_reader(BufReader::new(file)) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Could not load cache from json, falling back to default. Error: {}",
|
||||
e
|
||||
);
|
||||
return Cache {
|
||||
file: Some(path.as_ref().to_path_buf()),
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
};
|
||||
Cache {
|
||||
data: Arc::new(Mutex::new(data)),
|
||||
file: Some(path.as_ref().to_path_buf()),
|
||||
}
|
||||
}
|
||||
|
||||
fn save(&self, cache: &CacheData) {
|
||||
match self.file.as_ref() {
|
||||
Some(file) => match File::create(file) {
|
||||
Ok(file) => match serde_json::to_writer(file, cache) {
|
||||
Ok(_) => {}
|
||||
Err(e) => error!("Could not write cache to json. Error: {}", e),
|
||||
},
|
||||
Err(e) => error!("Could not open cache json file. Error: {}", e),
|
||||
},
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use temp_testdir::TempDir;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test() {
|
||||
let cache = Cache::default();
|
||||
|
||||
let desktop_c = cache
|
||||
.get_desktop_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "1.2.3".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
desktop_c,
|
||||
ClientData {
|
||||
version: "1.2.3".to_owned()
|
||||
}
|
||||
);
|
||||
|
||||
let music_c = cache
|
||||
.get_music_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "4.5.6".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
music_c,
|
||||
ClientData {
|
||||
version: "4.5.6".to_owned()
|
||||
}
|
||||
);
|
||||
|
||||
let deobf_data = cache
|
||||
.get_deobf_data(async {
|
||||
Ok(DeobfData {
|
||||
js_url:
|
||||
"https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
|
||||
.to_owned(),
|
||||
sig_fn: "t_sig_fn".to_owned(),
|
||||
nsig_fn: "t_nsig_fn".to_owned(),
|
||||
sts: "t_sts".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
deobf_data,
|
||||
DeobfData {
|
||||
js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
|
||||
.to_owned(),
|
||||
sig_fn: "t_sig_fn".to_owned(),
|
||||
nsig_fn: "t_nsig_fn".to_owned(),
|
||||
sts: "t_sts".to_owned(),
|
||||
}
|
||||
);
|
||||
|
||||
// Create a new cache from the first one's json
|
||||
// and check if it returns the same cached data
|
||||
let json = cache.to_json().await.unwrap();
|
||||
let new_cache = Cache::from_json(&json);
|
||||
|
||||
assert_eq!(
|
||||
new_cache
|
||||
.get_desktop_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
desktop_c
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
new_cache
|
||||
.get_music_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
music_c
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
new_cache
|
||||
.get_deobf_data(async {
|
||||
Ok(DeobfData {
|
||||
js_url: "".to_owned(),
|
||||
nsig_fn: "".to_owned(),
|
||||
sig_fn: "".to_owned(),
|
||||
sts: "".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
deobf_data
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file() {
|
||||
let temp = TempDir::default();
|
||||
let mut file_path = PathBuf::from(temp.as_ref());
|
||||
file_path.push("cache.json");
|
||||
|
||||
let cache = Cache::from_json_file(file_path.clone());
|
||||
|
||||
let cdata = cache
|
||||
.get_desktop_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "1.2.3".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let deobf_data = cache
|
||||
.get_deobf_data(async {
|
||||
Ok(DeobfData {
|
||||
js_url:
|
||||
"https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
|
||||
.to_owned(),
|
||||
sig_fn: "t_sig_fn".to_owned(),
|
||||
nsig_fn: "t_nsig_fn".to_owned(),
|
||||
sts: "t_sts".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(file_path.exists());
|
||||
let new_cache = Cache::from_json_file(file_path.clone());
|
||||
|
||||
assert_eq!(
|
||||
new_cache
|
||||
.get_desktop_client_data(async {
|
||||
Ok(ClientData {
|
||||
version: "".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
cdata
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
new_cache
|
||||
.get_deobf_data(async {
|
||||
Ok(DeobfData {
|
||||
js_url: "".to_owned(),
|
||||
nsig_fn: "".to_owned(),
|
||||
sig_fn: "".to_owned(),
|
||||
sts: "".to_owned(),
|
||||
})
|
||||
})
|
||||
.await
|
||||
.unwrap(),
|
||||
deobf_data
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,20 +6,35 @@ mod response;
|
|||
use std::fmt::Debug;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use fancy_regex::Regex;
|
||||
use log::{error, warn};
|
||||
use once_cell::sync::Lazy;
|
||||
use rand::Rng;
|
||||
use reqwest::{header, Client, ClientBuilder, Method, RequestBuilder};
|
||||
use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::{
|
||||
cache::Cache,
|
||||
deobfuscate::Deobfuscator,
|
||||
cache::{CacheStorage, FileStorage},
|
||||
deobfuscate::{DeobfData, Deobfuscator},
|
||||
model::{Country, Language},
|
||||
report::{Level, Report, Reporter, YamlFileReporter},
|
||||
report::{JsonFileReporter, Level, Report, Reporter},
|
||||
util,
|
||||
};
|
||||
|
||||
/// Client types for accessing the YouTube API.
|
||||
///
|
||||
/// There are multiple clients for accessing the YouTube API which have
|
||||
/// slightly different features
|
||||
///
|
||||
/// - **Desktop**: used by youtube.com
|
||||
/// - **DesktopMusic**: used by music.youtube.com, can access special music data,
|
||||
/// cannot access non-music content
|
||||
/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos
|
||||
/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams
|
||||
/// - **Ios**: used by the iOS app, no obfuscated URLs
|
||||
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ClientType {
|
||||
|
|
@ -95,7 +110,7 @@ impl Default for RequestYT {
|
|||
#[derive(Clone, Debug, Serialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct User {
|
||||
// TO DO: provide a way to enable restricted mode with:
|
||||
// TODO: provide a way to enable restricted mode with:
|
||||
// "enableSafetyMode": true
|
||||
locked_safety_mode: bool,
|
||||
}
|
||||
|
|
@ -131,6 +146,11 @@ const IOS_DEVICE_MODEL: &str = "iPhone14,5";
|
|||
static CLIENT_VERSION_REGEXES: Lazy<[Regex; 1]> =
|
||||
Lazy::new(|| [Regex::new("INNERTUBE_CONTEXT_CLIENT_VERSION\":\"([0-9\\.]+?)\"").unwrap()]);
|
||||
|
||||
/// The RustyPipe client used to access YouTube's API
|
||||
///
|
||||
/// RustyPipe includes an `Arc` internally, so if you are using the client
|
||||
/// at multiple locations, you can just clone it. Note that options (lang/country/report)
|
||||
/// are not shared between clones.
|
||||
#[derive(Clone)]
|
||||
pub struct RustyPipe {
|
||||
inner: Arc<RustyPipeRef>,
|
||||
|
|
@ -139,10 +159,11 @@ pub struct RustyPipe {
|
|||
|
||||
struct RustyPipeRef {
|
||||
http: Client,
|
||||
cache: Cache,
|
||||
storage: Option<Box<dyn CacheStorage>>,
|
||||
reporter: Option<Box<dyn Reporter>>,
|
||||
user_agent: String,
|
||||
consent_cookie: String,
|
||||
cache: Mutex<CacheData>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -150,13 +171,14 @@ struct RustyPipeOpts {
|
|||
lang: Language,
|
||||
country: Country,
|
||||
report: bool,
|
||||
strict: bool,
|
||||
}
|
||||
|
||||
impl Default for RustyPipe {
|
||||
fn default() -> Self {
|
||||
Self::new(
|
||||
Some(Cache::from_json_file("RustyPipeCache.json")),
|
||||
Some(Box::new(YamlFileReporter::default())),
|
||||
Some(Box::new(FileStorage::default())),
|
||||
Some(Box::new(JsonFileReporter::default())),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
|
@ -168,17 +190,64 @@ impl Default for RustyPipeOpts {
|
|||
lang: Language::En,
|
||||
country: Country::Us,
|
||||
report: false,
|
||||
strict: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
|
||||
struct CacheData {
|
||||
desktop_client: CacheEntry<ClientData>,
|
||||
music_client: CacheEntry<ClientData>,
|
||||
deobf: CacheEntry<DeobfData>,
|
||||
}
|
||||
|
||||
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
|
||||
enum CacheEntry<T> {
|
||||
#[default]
|
||||
None,
|
||||
Some {
|
||||
last_update: DateTime<Utc>,
|
||||
data: T,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ClientData {
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
impl<T> CacheEntry<T> {
|
||||
fn get(&self) -> Option<&T> {
|
||||
match self {
|
||||
CacheEntry::Some { last_update, data } => {
|
||||
if last_update < &(Utc::now() - Duration::hours(24)) {
|
||||
None
|
||||
} else {
|
||||
Some(data)
|
||||
}
|
||||
}
|
||||
CacheEntry::None => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<T> for CacheEntry<T> {
|
||||
fn from(f: T) -> Self {
|
||||
Self::Some {
|
||||
last_update: Utc::now(),
|
||||
data: f,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RustyPipe {
|
||||
/// Create a new RustyPipe instance
|
||||
pub fn new(
|
||||
cache: Option<Cache>,
|
||||
storage: Option<Box<dyn CacheStorage>>,
|
||||
reporter: Option<Box<dyn Reporter>>,
|
||||
user_agent: Option<String>,
|
||||
) -> Self {
|
||||
let cache = cache.unwrap_or_else(|| Cache::default());
|
||||
let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned());
|
||||
|
||||
let http = ClientBuilder::new()
|
||||
|
|
@ -188,10 +257,26 @@ impl RustyPipe {
|
|||
.build()
|
||||
.expect("unable to build the HTTP client");
|
||||
|
||||
let cache = if let Some(storage) = &storage {
|
||||
if let Some(data) = storage.read() {
|
||||
match serde_json::from_str::<CacheData>(&data) {
|
||||
Ok(data) => data,
|
||||
Err(e) => {
|
||||
error!("Could not deserialize cache. Error: {}", e);
|
||||
CacheData::default()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
CacheData::default()
|
||||
}
|
||||
} else {
|
||||
CacheData::default()
|
||||
};
|
||||
|
||||
RustyPipe {
|
||||
inner: Arc::new(RustyPipeRef {
|
||||
http,
|
||||
cache,
|
||||
storage,
|
||||
reporter,
|
||||
user_agent,
|
||||
consent_cookie: format!(
|
||||
|
|
@ -200,26 +285,53 @@ impl RustyPipe {
|
|||
CONSENT_COOKIE_YES,
|
||||
rand::thread_rng().gen_range(100..1000)
|
||||
),
|
||||
cache: Mutex::new(cache),
|
||||
}),
|
||||
opts: RustyPipeOpts::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new RustyPipe instance configured for testing
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "yaml")]
|
||||
pub fn new_test() -> Self {
|
||||
Self::new(
|
||||
Some(Box::new(FileStorage::default())),
|
||||
Some(Box::new(crate::report::YamlFileReporter::default())),
|
||||
None,
|
||||
)
|
||||
.strict(true)
|
||||
}
|
||||
|
||||
/// Set the language parameter used when accessing the YouTube API
|
||||
/// This will change multilanguage video titles, descriptions and textual dates
|
||||
pub fn lang(mut self, lang: Language) -> Self {
|
||||
self.opts.lang = lang;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the country parameter used when accessing the YouTube API.
|
||||
/// This will change trends and recommended content.
|
||||
pub fn country(mut self, country: Country) -> Self {
|
||||
self.opts.country = country;
|
||||
self
|
||||
}
|
||||
|
||||
/// Generate a report on every operation.
|
||||
/// This should only be used for debugging.
|
||||
pub fn report(mut self, report: bool) -> Self {
|
||||
self.opts.report = report;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable strict mode, causing operations to fail if there
|
||||
/// are warnings during deserialization (e.g. invalid items).
|
||||
/// This should only be used for testing.
|
||||
pub fn strict(mut self, strict: bool) -> Self {
|
||||
self.opts.strict = strict;
|
||||
self
|
||||
}
|
||||
|
||||
async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT {
|
||||
let hl = match localized {
|
||||
true => self.opts.lang,
|
||||
|
|
@ -234,7 +346,7 @@ impl RustyPipe {
|
|||
ClientType::Desktop => ContextYT {
|
||||
client: ClientInfo {
|
||||
client_name: "WEB".to_owned(),
|
||||
client_version: DESKTOP_CLIENT_VERSION.to_owned(),
|
||||
client_version: self.get_desktop_client_version().await,
|
||||
client_screen: None,
|
||||
device_model: None,
|
||||
platform: "DESKTOP".to_owned(),
|
||||
|
|
@ -249,7 +361,7 @@ impl RustyPipe {
|
|||
ClientType::DesktopMusic => ContextYT {
|
||||
client: ClientInfo {
|
||||
client_name: "WEB_REMIX".to_owned(),
|
||||
client_version: DESKTOP_MUSIC_CLIENT_VERSION.to_owned(),
|
||||
client_version: self.get_music_client_version().await,
|
||||
client_screen: None,
|
||||
device_model: None,
|
||||
platform: "DESKTOP".to_owned(),
|
||||
|
|
@ -332,7 +444,7 @@ impl RustyPipe {
|
|||
.header(header::REFERER, "https://www.youtube.com")
|
||||
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
|
||||
.header("X-YouTube-Client-Name", "1")
|
||||
.header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION),
|
||||
.header("X-YouTube-Client-Version", self.get_desktop_client_version().await),
|
||||
ClientType::DesktopMusic => self
|
||||
.inner
|
||||
.http
|
||||
|
|
@ -350,7 +462,7 @@ impl RustyPipe {
|
|||
.header(header::REFERER, "https://music.youtube.com")
|
||||
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
|
||||
.header("X-YouTube-Client-Name", "67")
|
||||
.header("X-YouTube-Client-Version", DESKTOP_MUSIC_CLIENT_VERSION),
|
||||
.header("X-YouTube-Client-Version", self.get_music_client_version().await),
|
||||
ClientType::TvHtml5Embed => self
|
||||
.inner
|
||||
.http
|
||||
|
|
@ -410,7 +522,7 @@ impl RustyPipe {
|
|||
}
|
||||
}
|
||||
|
||||
async fn execute_request<
|
||||
async fn execute_request_deobf<
|
||||
R: DeserializeOwned + MapResponse<M> + Debug,
|
||||
M,
|
||||
B: Serialize + ?Sized,
|
||||
|
|
@ -448,6 +560,7 @@ impl RustyPipe {
|
|||
operation: operation.to_owned(),
|
||||
error,
|
||||
msgs,
|
||||
deobf_data: deobf.map(Deobfuscator::get_data),
|
||||
http_request: crate::report::HTTPRequest {
|
||||
url: request_url,
|
||||
method: method.to_string(),
|
||||
|
|
@ -482,6 +595,10 @@ impl RustyPipe {
|
|||
Some("Warnings during deserialization/mapping".to_owned()),
|
||||
mapres.warnings,
|
||||
);
|
||||
|
||||
if self.opts.strict {
|
||||
bail!("Warnings during deserialization/mapping");
|
||||
}
|
||||
} else if self.opts.report {
|
||||
create_report(Level::DBG, None, vec![]);
|
||||
}
|
||||
|
|
@ -500,6 +617,176 @@ impl RustyPipe {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_request<
|
||||
R: DeserializeOwned + MapResponse<M> + Debug,
|
||||
M,
|
||||
B: Serialize + ?Sized,
|
||||
>(
|
||||
&self,
|
||||
ctype: ClientType,
|
||||
operation: &str,
|
||||
method: Method,
|
||||
endpoint: &str,
|
||||
id: &str,
|
||||
body: &B,
|
||||
) -> Result<M> {
|
||||
self.execute_request_deobf::<R, M, B>(ctype, operation, method, endpoint, id, body, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_desktop_client_version(&self) -> String {
|
||||
let mut cache = self.inner.cache.lock().await;
|
||||
|
||||
match cache.desktop_client.get() {
|
||||
Some(cdata) => cdata.version.to_owned(),
|
||||
None => match self.extract_desktop_client_version().await {
|
||||
Ok(version) => {
|
||||
cache.desktop_client = CacheEntry::from(ClientData {
|
||||
version: version.to_owned(),
|
||||
});
|
||||
self.write_cache(&cache);
|
||||
version
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("{}, falling back to hardcoded version", e);
|
||||
DESKTOP_CLIENT_VERSION.to_owned()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_music_client_version(&self) -> String {
|
||||
let mut cache = self.inner.cache.lock().await;
|
||||
|
||||
match cache.music_client.get() {
|
||||
Some(cdata) => cdata.version.to_owned(),
|
||||
None => match self.extract_music_client_version().await {
|
||||
Ok(version) => {
|
||||
cache.music_client = CacheEntry::from(ClientData {
|
||||
version: version.to_owned(),
|
||||
});
|
||||
self.write_cache(&cache);
|
||||
version
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("{}, falling back to hardcoded version", e);
|
||||
DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_deobf(&self) -> Result<Deobfuscator> {
|
||||
let mut cache = self.inner.cache.lock().await;
|
||||
let deobf = Deobfuscator::new(self.inner.http.clone()).await?;
|
||||
cache.deobf = CacheEntry::from(deobf.get_data());
|
||||
self.write_cache(&cache);
|
||||
Ok(deobf)
|
||||
}
|
||||
|
||||
async fn extract_desktop_client_version(&self) -> Result<String> {
|
||||
let from_swjs = async {
|
||||
let swjs = self
|
||||
.exec_request_text(
|
||||
self.inner
|
||||
.http
|
||||
.get("https://www.youtube.com/sw.js")
|
||||
.header(header::ORIGIN, "https://www.youtube.com")
|
||||
.header(header::REFERER, "https://www.youtube.com")
|
||||
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.context("Failed to download sw.js")?;
|
||||
|
||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
|
||||
.ok_or(anyhow!("Could not find desktop client version in sw.js"))
|
||||
};
|
||||
|
||||
let from_html = async {
|
||||
let html = self
|
||||
.exec_request_text(
|
||||
self.inner
|
||||
.http
|
||||
.get("https://www.youtube.com/results?search_query=")
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.context("Failed to get YT Desktop page")?;
|
||||
|
||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!(
|
||||
"Could not find desktop client version on html page"
|
||||
))
|
||||
};
|
||||
|
||||
match from_swjs.await {
|
||||
Ok(client_version) => Ok(client_version),
|
||||
Err(_) => from_html.await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn extract_music_client_version(&self) -> Result<String> {
|
||||
let from_swjs = async {
|
||||
let swjs = self
|
||||
.exec_request_text(
|
||||
self.inner
|
||||
.http
|
||||
.get("https://music.youtube.com/sw.js")
|
||||
.header(header::ORIGIN, "https://music.youtube.com")
|
||||
.header(header::REFERER, "https://music.youtube.com")
|
||||
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.context("Failed to download sw.js")?;
|
||||
|
||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
|
||||
.ok_or(anyhow!("Could not find desktop client version in sw.js"))
|
||||
};
|
||||
|
||||
let from_html = async {
|
||||
let html = self
|
||||
.exec_request_text(
|
||||
self.inner
|
||||
.http
|
||||
.get("https://music.youtube.com")
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.context("Failed to get YT Desktop page")?;
|
||||
|
||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(anyhow!(
|
||||
"Could not find desktop client version on html page"
|
||||
))
|
||||
};
|
||||
|
||||
match from_swjs.await {
|
||||
Ok(client_version) => Ok(client_version),
|
||||
Err(_) => from_html.await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn exec_request(&self, request: Request) -> Result<Response> {
|
||||
Ok(self.inner.http.execute(request).await?.error_for_status()?)
|
||||
}
|
||||
|
||||
async fn exec_request_text(&self, request: Request) -> Result<String> {
|
||||
Ok(self.exec_request(request).await?.text().await?)
|
||||
}
|
||||
|
||||
fn write_cache(&self, cache: &CacheData) {
|
||||
if let Some(storage) = &self.inner.storage {
|
||||
match serde_json::to_string(cache) {
|
||||
Ok(data) => storage.write(&data),
|
||||
Err(e) => error!("Could not serialize cache. Error: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
trait MapResponse<T> {
|
||||
|
|
@ -525,10 +812,3 @@ where
|
|||
self.c.fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ struct QContentPlaybackContext {
|
|||
|
||||
impl RustyPipe {
|
||||
pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
|
||||
let (context, deobf) = tokio::join!(
|
||||
self.get_context(client_type, false),
|
||||
Deobfuscator::from_fetched_info(self.inner.http.clone(), self.inner.cache.clone())
|
||||
);
|
||||
let (context, deobf) = tokio::join!(self.get_context(client_type, false), self.get_deobf());
|
||||
// let context = self.get_context(client_type, false).await;
|
||||
// let deobf = self.get_deobf().await;
|
||||
|
||||
let deobf = deobf?;
|
||||
|
||||
let request_body = if client_type.is_web() {
|
||||
|
|
@ -90,7 +90,7 @@ impl RustyPipe {
|
|||
}
|
||||
};
|
||||
|
||||
self.execute_request::<response::Player, _, _>(
|
||||
self.execute_request_deobf::<response::Player, _, _>(
|
||||
client_type,
|
||||
"get_player",
|
||||
Method::POST,
|
||||
|
|
@ -575,10 +575,11 @@ fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec {
|
|||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(feature = "yaml")]
|
||||
mod tests {
|
||||
use std::{fs::File, io::BufReader, path::Path};
|
||||
|
||||
use crate::{cache::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter};
|
||||
use crate::{deobfuscate::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter};
|
||||
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
|
@ -613,7 +614,7 @@ mod tests {
|
|||
#[test_log::test(tokio::test)]
|
||||
async fn download_model_testfiles() {
|
||||
let tf_dir = Path::new("testfiles/player_model");
|
||||
let rp = RustyPipe::default();
|
||||
let rp = RustyPipe::new_test();
|
||||
|
||||
for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] {
|
||||
let mut json_path = tf_dir.to_path_buf();
|
||||
|
|
@ -683,7 +684,7 @@ mod tests {
|
|||
#[case::ios(ClientType::Ios)]
|
||||
#[test_log::test(tokio::test)]
|
||||
async fn t_get_player(#[case] client_type: ClientType) {
|
||||
let rp = RustyPipe::default();
|
||||
let rp = RustyPipe::new_test();
|
||||
let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap();
|
||||
|
||||
// dbg!(&player_data);
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ impl RustyPipe {
|
|||
"browse",
|
||||
playlist_id,
|
||||
&request_body,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -62,7 +61,6 @@ impl RustyPipe {
|
|||
"browse",
|
||||
&playlist.id,
|
||||
&request_body,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -350,7 +348,7 @@ mod tests {
|
|||
#[case] description: Option<String>,
|
||||
#[case] channel: Option<Channel>,
|
||||
) {
|
||||
let rp = RustyPipe::default();
|
||||
let rp = RustyPipe::new_test();
|
||||
let playlist = rp.get_playlist(id).await.unwrap();
|
||||
|
||||
assert_eq!(playlist.id, id);
|
||||
|
|
@ -412,7 +410,7 @@ mod tests {
|
|||
|
||||
#[test_log::test(tokio::test)]
|
||||
async fn t_playlist_cont() {
|
||||
let rp = RustyPipe::default();
|
||||
let rp = RustyPipe::new_test();
|
||||
let mut playlist = rp
|
||||
.get_playlist("PLbZIPy20-1pN7mqjckepWF78ndb6ci_qi")
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -3,43 +3,47 @@ use fancy_regex::Regex;
|
|||
use log::debug;
|
||||
use once_cell::sync::Lazy;
|
||||
use reqwest::Client;
|
||||
use serde::{Serialize, Deserialize};
|
||||
use std::result::Result::Ok;
|
||||
|
||||
use crate::cache::{Cache, DeobfData};
|
||||
use crate::util;
|
||||
|
||||
pub struct Deobfuscator {
|
||||
data: DeobfData,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct DeobfData {
|
||||
pub js_url: String,
|
||||
pub sig_fn: String,
|
||||
pub nsig_fn: String,
|
||||
pub sts: String,
|
||||
}
|
||||
|
||||
impl Deobfuscator {
|
||||
pub async fn from_fetched_info(http: Client, cache: Cache) -> Result<Self> {
|
||||
let data = cache
|
||||
.get_deobf_data(async move {
|
||||
let js_url = get_player_js_url(&http)
|
||||
.await
|
||||
.context("Failed to retrieve player.js URL")?;
|
||||
pub async fn new(http: Client) -> Result<Self> {
|
||||
let js_url = get_player_js_url(&http)
|
||||
.await
|
||||
.context("Failed to retrieve player.js URL")?;
|
||||
|
||||
let player_js = get_response(&http, &js_url)
|
||||
.await
|
||||
.context("Failed to download player.js")?;
|
||||
let player_js = get_response(&http, &js_url)
|
||||
.await
|
||||
.context("Failed to download player.js")?;
|
||||
|
||||
debug!("Downloaded player.js from {}", js_url);
|
||||
debug!("Downloaded player.js from {}", js_url);
|
||||
|
||||
let sig_fn = get_sig_fn(&player_js)?;
|
||||
let nsig_fn = get_nsig_fn(&player_js)?;
|
||||
let sts = get_sts(&player_js)?;
|
||||
let sig_fn = get_sig_fn(&player_js)?;
|
||||
let nsig_fn = get_nsig_fn(&player_js)?;
|
||||
let sts = get_sts(&player_js)?;
|
||||
|
||||
Ok(DeobfData {
|
||||
js_url,
|
||||
nsig_fn,
|
||||
sig_fn,
|
||||
sts,
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(Self { data })
|
||||
Ok(Self {
|
||||
data: DeobfData {
|
||||
js_url,
|
||||
nsig_fn,
|
||||
sig_fn,
|
||||
sts,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
pub fn deobfuscate_sig(&self, sig: &str) -> Result<String> {
|
||||
|
|
@ -53,6 +57,10 @@ impl Deobfuscator {
|
|||
pub fn get_sts(&self) -> String {
|
||||
self.data.sts.to_owned()
|
||||
}
|
||||
|
||||
pub fn get_data(&self) -> DeobfData {
|
||||
self.data.to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<DeobfData> for Deobfuscator {
|
||||
|
|
@ -472,8 +480,7 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
|
|||
#[test(tokio::test)]
|
||||
async fn t_update() {
|
||||
let client = Client::new();
|
||||
let cache = Cache::default();
|
||||
let deobf = Deobfuscator::from_fetched_info(client, cache)
|
||||
let deobf = Deobfuscator::new(client)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ use chrono::{DateTime, Local};
|
|||
use log::error;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::deobfuscate::DeobfData;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Report {
|
||||
/// Rust package name (`rustypipe`)
|
||||
|
|
@ -25,9 +27,9 @@ pub struct Report {
|
|||
pub error: Option<String>,
|
||||
/// Detailed error/warning messages
|
||||
pub msgs: Vec<String>,
|
||||
// /// Deobfuscation data (only for player requests)
|
||||
// #[serde(skip_serializing_if = "Option::is_none")]
|
||||
// pub deobf_data: Option<DeobfData>,
|
||||
/// Deobfuscation data (only for player requests)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deobf_data: Option<DeobfData>,
|
||||
/// HTTP request data
|
||||
pub http_request: HTTPRequest,
|
||||
}
|
||||
|
|
@ -96,10 +98,12 @@ impl Reporter for JsonFileReporter {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature="yaml")]
|
||||
pub struct YamlFileReporter {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
#[cfg(feature="yaml")]
|
||||
impl YamlFileReporter {
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Self {
|
||||
Self {
|
||||
|
|
@ -114,6 +118,7 @@ impl YamlFileReporter {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature="yaml")]
|
||||
impl Default for YamlFileReporter {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
|
|
@ -122,6 +127,7 @@ impl Default for YamlFileReporter {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature="yaml")]
|
||||
impl Reporter for YamlFileReporter {
|
||||
fn report(&self, report: &Report) {
|
||||
self._report(report)
|
||||
|
|
|
|||
Reference in a new issue