Sulkta-OSS/rustypipe
Archived
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactored player

Browse source
This commit is contained in:
ThetaDev 2022-09-13 15:44:50 +02:00
parent 6cc927031a
commit 925652acdd
11 changed files with 1102 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

15
Cargo.toml
View file

@ -3,11 +3,16 @@ name = "rustypipe"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[workspace] [workspace]
members = [".", "cli"] members = [".", "cli"]
[features]
default = ["default-tls"]
default-tls = ["reqwest/default-tls"]
rustls-tls-webpki-roots = ["reqwest/rustls-tls-webpki-roots"]
rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
[dependencies] [dependencies]
# quick-js = "0.4.1" # quick-js = "0.4.1"
quick-js = { path = "../quickjs-rs" } quick-js = { path = "../quickjs-rs" }
@ -17,10 +22,11 @@ anyhow = "1.0"
thiserror = "1.0.31" thiserror = "1.0.31"
url = "2.2.2" url = "2.2.2"
log = "0.4.17" log = "0.4.17"
reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls-native-roots"]} reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]}
tokio = {version = "1.20.0", features = ["macros", "fs", "process"]} tokio = {version = "1.20.0", features = ["macros", "fs", "process"]}
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.82" serde_json = "1.0.82"
serde_yaml = "0.9.11"
serde_with = {version = "2.0.0", features = ["json"] } serde_with = {version = "2.0.0", features = ["json"] }
rand = "0.8.5" rand = "0.8.5"
async-trait = "0.1.56" async-trait = "0.1.56"
@ -31,13 +37,12 @@ indicatif = "0.17.0"
filenamify = "0.1.0" filenamify = "0.1.0"
ress = "0.11.4" ress = "0.11.4"
phf = "0.11.1" phf = "0.11.1"
serde_yaml = "0.9.11"
[dev-dependencies] [dev-dependencies]
env_logger = "0.9.0" env_logger = "0.9.0"
test-log = "0.2.11" test-log = "0.2.11"
rstest = "0.15.0" rstest = "0.15.0"
temp_testdir = "0.2.3" temp_testdir = "0.2.3"
insta = {version = "1.17.1", features = ["redactions"]} insta = {version = "1.17.1", features = ["yaml", "redactions"]}
velcro = "0.5.3" velcro = "0.5.3"
phf_codegen = "0.11.1" phf_codegen = "0.11.1"

6
cli/Cargo.toml
View file

@ -3,11 +3,9 @@ name = "rustypipe-cli"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
rustypipe = {path = "../"} rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]}
reqwest = {version = "0.11.11", default_features = false, features = ["gzip", "brotli", "rustls-tls-native-roots"]} reqwest = {version = "0.11.11", default_features = false}
tokio = {version = "1.20.0", features = ["rt-multi-thread"]} tokio = {version = "1.20.0", features = ["rt-multi-thread"]}
indicatif = "0.17.0" indicatif = "0.17.0"
futures = "0.3.21" futures = "0.3.21"

16
cli/src/main.rs
View file

@ -6,7 +6,7 @@ use futures::stream::{self, StreamExt};
use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use reqwest::{Client, ClientBuilder}; use reqwest::{Client, ClientBuilder};
use rustypipe::{ use rustypipe::{
client::{ClientType, RustyTube}, client2::{ClientType, RustyPipe},
model::stream_filter::Filter, model::stream_filter::Filter,
}; };
@ -46,7 +46,7 @@ async fn download_single_video(
output_fname: Option<String>, output_fname: Option<String>,
resolution: Option<u32>, resolution: Option<u32>,
ffmpeg: &str, ffmpeg: &str,
rt: &RustyTube, rp: &RustyPipe,
http: Client, http: Client,
multi: MultiProgress, multi: MultiProgress,
main: Option<ProgressBar>, main: Option<ProgressBar>,
@ -58,7 +58,7 @@ async fn download_single_video(
pb.set_message(format!("Fetching player data for {}", video_title)); pb.set_message(format!("Fetching player data for {}", video_title));
let res = async { let res = async {
let player_data = rt let player_data = rp
.get_player(video_id.as_str(), ClientType::TvHtml5Embed) .get_player(video_id.as_str(), ClientType::TvHtml5Embed)
.await .await
.context(format!( .context(format!(
@ -112,7 +112,7 @@ async fn download_video(
.build() .build()
.expect("unable to build the HTTP client"); .expect("unable to build the HTTP client");
let rt = RustyTube::new(); let rp = RustyPipe::default();
// Indicatif setup // Indicatif setup
let multi = MultiProgress::new(); let multi = MultiProgress::new();
@ -124,7 +124,7 @@ async fn download_video(
output_fname, output_fname,
resolution, resolution,
"ffmpeg", "ffmpeg",
&rt, &rp,
http, http,
multi, multi,
None, None,
@ -147,8 +147,8 @@ async fn download_playlist(
.build() .build()
.expect("unable to build the HTTP client"); .expect("unable to build the HTTP client");
let rt = RustyTube::new(); let rp = RustyPipe::default();
let playlist = rt.get_playlist(id).await.unwrap(); let playlist = rp.get_playlist(id).await.unwrap();
// Indicatif setup // Indicatif setup
let multi = MultiProgress::new(); let multi = MultiProgress::new();
@ -173,7 +173,7 @@ async fn download_playlist(
output_fname.to_owned(), output_fname.to_owned(),
resolution, resolution,
"ffmpeg", "ffmpeg",
&rt, &rp,
http.clone(), http.clone(),
multi.clone(), multi.clone(),
Some(main.clone()), Some(main.clone()),

272
src/client2/mod.rs
View file

@ -1,3 +1,4 @@
pub mod player;
pub mod playlist; pub mod playlist;
mod response; mod response;
@ -14,8 +15,9 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
use crate::{ use crate::{
cache::Cache, cache::Cache,
deobfuscate::Deobfuscator,
model::{Country, Language}, model::{Country, Language},
report::{YamlFileReporter, Level, Report, Reporter}, report::{Level, Report, Reporter, YamlFileReporter},
}; };
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
@ -28,6 +30,23 @@ pub enum ClientType {
Ios, Ios,
} }
const CLIENT_TYPES: [ClientType; 5] = [
ClientType::Desktop,
ClientType::DesktopMusic,
ClientType::TvHtml5Embed,
ClientType::Android,
ClientType::Ios,
];
impl ClientType {
fn is_web(&self) -> bool {
match self {
ClientType::Desktop | ClientType::DesktopMusic | ClientType::TvHtml5Embed => true,
ClientType::Android | ClientType::Ios => false,
}
}
}
#[derive(Clone, Debug, Serialize)] #[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct ContextYT { pub struct ContextYT {
@ -163,6 +182,7 @@ impl RustyPipe {
let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned()); let user_agent = user_agent.unwrap_or(DEFAULT_UA.to_owned());
let http = ClientBuilder::new() let http = ClientBuilder::new()
.user_agent(user_agent.to_owned())
.gzip(true) .gzip(true)
.brotli(true) .brotli(true)
.build() .build()
@ -201,6 +221,15 @@ impl RustyPipe {
} }
async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT { async fn get_context(&self, ctype: ClientType, localized: bool) -> ContextYT {
let hl = match localized {
true => self.opts.lang,
false => Language::En,
};
let gl = match localized {
true => self.opts.country,
false => Country::Us,
};
match ctype { match ctype {
ClientType::Desktop => ContextYT { ClientType::Desktop => ContextYT {
client: ClientInfo { client: ClientInfo {
@ -210,23 +239,75 @@ impl RustyPipe {
device_model: None, device_model: None,
platform: "DESKTOP".to_owned(), platform: "DESKTOP".to_owned(),
original_url: Some("https://www.youtube.com/".to_owned()), original_url: Some("https://www.youtube.com/".to_owned()),
hl: match localized { hl,
true => self.opts.lang, gl,
false => Language::En,
},
gl: match localized {
true => self.opts.country,
false => Country::Us,
},
}, },
request: Some(RequestYT::default()), request: Some(RequestYT::default()),
user: User::default(), user: User::default(),
third_party: None, third_party: None,
}, },
ClientType::DesktopMusic => todo!(), ClientType::DesktopMusic => ContextYT {
ClientType::TvHtml5Embed => todo!(), client: ClientInfo {
ClientType::Android => todo!(), client_name: "WEB_REMIX".to_owned(),
ClientType::Ios => todo!(), client_version: DESKTOP_MUSIC_CLIENT_VERSION.to_owned(),
client_screen: None,
device_model: None,
platform: "DESKTOP".to_owned(),
original_url: Some("https://music.youtube.com/".to_owned()),
hl,
gl,
},
request: Some(RequestYT::default()),
user: User::default(),
third_party: None,
},
ClientType::TvHtml5Embed => ContextYT {
client: ClientInfo {
client_name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER".to_owned(),
client_version: TVHTML5_CLIENT_VERSION.to_owned(),
client_screen: Some("EMBED".to_owned()),
device_model: None,
platform: "TV".to_owned(),
original_url: None,
hl,
gl,
},
request: Some(RequestYT::default()),
user: User::default(),
third_party: Some(ThirdParty {
embed_url: "https://www.youtube.com/".to_owned(),
}),
},
ClientType::Android => ContextYT {
client: ClientInfo {
client_name: "ANDROID".to_owned(),
client_version: MOBILE_CLIENT_VERSION.to_owned(),
client_screen: None,
device_model: None,
platform: "MOBILE".to_owned(),
original_url: None,
hl,
gl,
},
request: None,
user: User::default(),
third_party: None,
},
ClientType::Ios => ContextYT {
client: ClientInfo {
client_name: "IOS".to_owned(),
client_version: MOBILE_CLIENT_VERSION.to_owned(),
client_screen: None,
device_model: Some(IOS_DEVICE_MODEL.to_owned()),
platform: "MOBILE".to_owned(),
original_url: None,
hl,
gl,
},
request: None,
user: User::default(),
third_party: None,
},
} }
} }
@ -252,10 +333,80 @@ impl RustyPipe {
.header(header::COOKIE, self.inner.consent_cookie.to_owned()) .header(header::COOKIE, self.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "1") .header("X-YouTube-Client-Name", "1")
.header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION), .header("X-YouTube-Client-Version", DESKTOP_CLIENT_VERSION),
ClientType::DesktopMusic => todo!(), ClientType::DesktopMusic => self
ClientType::TvHtml5Embed => todo!(), .inner
ClientType::Android => todo!(), .http
ClientType::Ios => todo!(), .request(
method,
format!(
"{}{}?key={}{}",
YOUTUBE_MUSIC_V1_URL,
endpoint,
DESKTOP_MUSIC_API_KEY,
DISABLE_PRETTY_PRINT_PARAMETER
),
)
.header(header::ORIGIN, "https://music.youtube.com")
.header(header::REFERER, "https://music.youtube.com")
.header(header::COOKIE, self.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "67")
.header("X-YouTube-Client-Version", DESKTOP_MUSIC_CLIENT_VERSION),
ClientType::TvHtml5Embed => self
.inner
.http
.request(
method,
format!(
"{}{}?key={}{}",
YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER
),
)
.header(header::ORIGIN, "https://www.youtube.com")
.header(header::REFERER, "https://www.youtube.com")
.header("X-YouTube-Client-Name", "1")
.header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION),
ClientType::Android => self
.inner
.http
.request(
method,
format!(
"{}{}?key={}{}",
YOUTUBEI_V1_GAPIS_URL,
endpoint,
ANDROID_API_KEY,
DISABLE_PRETTY_PRINT_PARAMETER
),
)
.header(
header::USER_AGENT,
format!(
"com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip",
MOBILE_CLIENT_VERSION, self.opts.country
),
)
.header("X-Goog-Api-Format-Version", "2"),
ClientType::Ios => self
.inner
.http
.request(
method,
format!(
"{}{}?key={}{}",
YOUTUBEI_V1_GAPIS_URL,
endpoint,
IOS_API_KEY,
DISABLE_PRETTY_PRINT_PARAMETER
),
)
.header(
header::USER_AGENT,
format!(
"com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})",
MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, self.opts.country
),
)
.header("X-Goog-Api-Format-Version", "2"),
} }
} }
@ -271,6 +422,7 @@ impl RustyPipe {
endpoint: &str, endpoint: &str,
id: &str, id: &str,
body: &B, body: &B,
deobf: Option<&Deobfuscator>,
) -> Result<M> { ) -> Result<M> {
let request = self let request = self
.request_builder(ctype, method.clone(), endpoint) .request_builder(ctype, method.clone(), endpoint)
@ -286,80 +438,77 @@ impl RustyPipe {
let status = response.status(); let status = response.status();
let resp_str = response.text().await?; let resp_str = response.text().await?;
let create_report = let create_report = |level: Level, error: Option<String>, msgs: Vec<String>| {
|level: Level, error: Option<String>, msgs: Vec<String>, deserialized: Option<&R>| { if let Some(reporter) = &self.inner.reporter {
if let Some(reporter) = &self.inner.reporter { let report = Report {
let report = Report { package: "rustypipe".to_owned(),
package: "rustypipe".to_owned(), version: "0.1.0".to_owned(),
version: "0.1.0".to_owned(), date: chrono::Local::now(),
date: chrono::Local::now(), level,
level, operation: operation.to_owned(),
operation: operation.to_owned(), error,
error, msgs,
msgs, http_request: crate::report::HTTPRequest {
http_request: crate::report::HTTPRequest { url: request_url,
url: request_url, method: method.to_string(),
method: method.to_string(), req_header: request_headers
req_header: request_headers .iter()
.iter() .map(|(k, v)| {
.map(|(k, v)| { (k.to_string(), v.to_str().unwrap_or_default().to_owned())
(k.to_string(), v.to_str().unwrap_or_default().to_owned()) })
}) .collect(),
.collect(), req_body: serde_json::to_string(body).unwrap_or_default(),
req_body: serde_json::to_string(body).unwrap_or_default(), status: status.into(),
status: status.into(), resp_body: resp_str.to_owned(),
resp_body: resp_str.to_owned(), },
}, };
deserialized: deserialized.map(|d| format!("{:?}", d)),
};
reporter.report(&report); reporter.report(&report);
} }
}; };
if status.is_client_error() || status.is_server_error() { if status.is_client_error() || status.is_server_error() {
let e = anyhow!("Server responded with error code {}", status); let e = anyhow!("Server responded with error code {}", status);
create_report(Level::ERR, Some(e.to_string()), vec![], None); create_report(Level::ERR, Some(e.to_string()), vec![]);
return Err(e); return Err(e);
} }
match serde_json::from_str::<R>(&resp_str) { match serde_json::from_str::<R>(&resp_str) {
Ok(deserialized) => match deserialized.map_response(self.opts.lang, id) { Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) {
Ok(mapres) => { Ok(mapres) => {
if !mapres.warnings.is_empty() { if !mapres.warnings.is_empty() {
create_report( create_report(
Level::WRN, Level::WRN,
Some("Warnings during deserialization/mapping".to_owned()), Some("Warnings during deserialization/mapping".to_owned()),
mapres.warnings, mapres.warnings,
Some(&deserialized),
); );
} else if self.opts.report { } else if self.opts.report {
create_report(Level::DBG, None, vec![], Some(&deserialized)); create_report(Level::DBG, None, vec![]);
} }
Ok(mapres.c) Ok(mapres.c)
} }
Err(e) => { Err(e) => {
let emsg = "Could not map reponse"; let emsg = "Could not map reponse";
create_report( create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
Level::ERR,
Some(emsg.to_owned()),
vec![e.to_string()],
Some(&deserialized),
);
Err(e).context(emsg) Err(e).context(emsg)
} }
}, },
Err(e) => { Err(e) => {
let emsg = "Could not deserialize response"; let emsg = "Could not deserialize response";
create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()], None); create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
Err(e).context(emsg) Err(e).context(emsg)
} }
} }
} }
} }
pub trait MapResponse<T> { trait MapResponse<T> {
fn map_response(&self, lang: Language, id: &str) -> Result<MapResult<T>>; fn map_response(
self,
id: &str,
lang: Language,
deobf: Option<&Deobfuscator>,
) -> Result<MapResult<T>>;
} }
#[derive(Clone)] #[derive(Clone)]
@ -368,7 +517,10 @@ pub struct MapResult<T> {
pub warnings: Vec<String>, pub warnings: Vec<String>,
} }
impl<T> Debug for MapResult<T> where T: Debug { impl<T> Debug for MapResult<T>
where
T: Debug,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.c.fmt(f) self.c.fmt(f)
} }

807
src/client2/player.rs Normal file
View file

@ -0,0 +1,807 @@
use std::{
borrow::Cow,
collections::{BTreeMap, HashMap},
};
use anyhow::{anyhow, bail, Result};
use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use reqwest::{Method, Url};
use serde::Serialize;
use crate::{
deobfuscate::Deobfuscator,
model::{
AudioCodec, AudioFormat, AudioStream, AudioTrack, Channel, Language, Subtitle, VideoCodec,
VideoFormat, VideoInfo, VideoPlayer, VideoStream,
},
util,
};
use super::{
response::{self, player},
ClientType, ContextYT, MapResponse, MapResult, RustyPipe,
};
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct QPlayer {
context: ContextYT,
/// Website playback context
#[serde(skip_serializing_if = "Option::is_none")]
playback_context: Option<QPlaybackContext>,
/// Content playback nonce (mobile only, 16 random chars)
#[serde(skip_serializing_if = "Option::is_none")]
cpn: Option<String>,
/// YouTube video ID
video_id: String,
/// Set to true to allow extraction of streams with sensitive content
content_check_ok: bool,
/// Probably refers to allowing sensitive content, too
racy_check_ok: bool,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct QPlaybackContext {
content_playback_context: QContentPlaybackContext,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct QContentPlaybackContext {
/// Signature timestamp extracted from player.js
signature_timestamp: String,
/// Referer URL from website
referer: String,
}
impl RustyPipe {
pub async fn get_player(&self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
let (context, deobf) = tokio::join!(
self.get_context(client_type, false),
Deobfuscator::from_fetched_info(self.inner.http.clone(), self.inner.cache.clone())
);
let deobf = deobf?;
let request_body = if client_type.is_web() {
QPlayer {
context,
playback_context: Some(QPlaybackContext {
content_playback_context: QContentPlaybackContext {
signature_timestamp: deobf.get_sts(),
referer: format!("https://www.youtube.com/watch?v={}", video_id),
},
}),
cpn: None,
video_id: video_id.to_owned(),
content_check_ok: true,
racy_check_ok: true,
}
} else {
QPlayer {
context,
playback_context: None,
cpn: Some(util::generate_content_playback_nonce()),
video_id: video_id.to_owned(),
content_check_ok: true,
racy_check_ok: true,
}
};
self.execute_request::<response::Player, _, _>(
client_type,
"get_player",
Method::POST,
"player",
video_id,
&request_body,
Some(&deobf),
)
.await
}
}
impl MapResponse<VideoPlayer> for response::Player {
fn map_response(
self,
id: &str,
_lang: Language,
deobf: Option<&Deobfuscator>,
) -> Result<super::MapResult<VideoPlayer>> {
let deobf = deobf.unwrap();
let mut warnings = vec![];
// Check playability status
match self.playability_status {
response::player::PlayabilityStatus::Ok { live_streamability } => {
if live_streamability.is_some() {
bail!("Active livestreams are not supported")
}
}
response::player::PlayabilityStatus::Unplayable { reason } => {
bail!("Video is unplayable. Reason: {}", reason)
}
response::player::PlayabilityStatus::LoginRequired { reason } => {
bail!("Playback requires login. Reason: {}", reason)
}
response::player::PlayabilityStatus::LiveStreamOffline { reason } => {
bail!("Livestream is offline. Reason: {}", reason)
}
response::player::PlayabilityStatus::Error { reason } => {
bail!("Video was deleted. Reason: {}", reason)
}
};
let mut streaming_data = some_or_bail!(
self.streaming_data,
Err(anyhow!("No streaming data was returned"))
);
let video_details = some_or_bail!(
self.video_details,
Err(anyhow!("No video details were returned"))
);
let microformat = self.microformat.map(|m| m.player_microformat_renderer);
let (publish_date, category, tags, is_family_safe) =
microformat.map_or((None, None, None, None), |m| {
(
Local
.from_local_datetime(&NaiveDateTime::new(
m.publish_date,
NaiveTime::from_hms(0, 0, 0),
))
.single(),
Some(m.category),
m.tags,
Some(m.is_family_safe),
)
});
if video_details.video_id != id {
bail!(
"got wrong video id {}, expected {}",
video_details.video_id,
id
);
}
let video_info = VideoInfo {
id: video_details.video_id,
title: video_details.title,
description: video_details.short_description,
length: video_details.length_seconds,
thumbnails: video_details.thumbnail.unwrap_or_default().into(),
channel: Channel {
id: video_details.channel_id,
name: video_details.author,
},
publish_date,
view_count: video_details.view_count,
keywords: match video_details.keywords {
Some(keywords) => keywords,
None => tags.unwrap_or_default(),
},
category,
is_live_content: video_details.is_live_content,
is_family_safe,
};
let mut formats = streaming_data.formats;
formats.append(&mut streaming_data.adaptive_formats);
let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()];
let mut video_streams: Vec<VideoStream> = Vec::new();
let mut video_only_streams: Vec<VideoStream> = Vec::new();
let mut audio_streams: Vec<AudioStream> = Vec::new();
for f in formats {
if f.format_type == player::FormatType::FormatStreamTypeOtf {
continue;
}
match (f.is_video(), f.is_audio()) {
(true, true) => {
let mut map_res = map_video_stream(f, deobf, &mut last_nsig);
warnings.append(&mut map_res.warnings);
if let Some(c) = map_res.c {
video_streams.push(c);
};
}
(true, false) => {
let mut map_res = map_video_stream(f, deobf, &mut last_nsig);
warnings.append(&mut map_res.warnings);
if let Some(c) = map_res.c {
video_only_streams.push(c);
};
}
(false, true) => {
let mut map_res = map_audio_stream(f, deobf, &mut last_nsig);
warnings.append(&mut map_res.warnings);
if let Some(c) = map_res.c {
audio_streams.push(c);
};
}
(false, false) => warnings.push(format!("invalid format: {}", f.itag)),
}
}
video_streams.sort();
video_only_streams.sort();
audio_streams.sort();
let mut subtitles = vec![];
if let Some(captions) = self.captions {
for c in captions.player_captions_tracklist_renderer.caption_tracks {
let lang_auto = c.name.strip_suffix(" (auto-generated)");
subtitles.push(Subtitle {
url: c.base_url,
lang: c.language_code,
lang_name: lang_auto.unwrap_or(&c.name).to_owned(),
auto_generated: lang_auto.is_some(),
})
}
}
Ok(MapResult {
c: VideoPlayer {
info: video_info,
video_streams,
video_only_streams,
audio_streams,
subtitles,
expires_in_seconds: streaming_data.expires_in_seconds,
},
warnings,
})
}
}
fn cipher_to_url_params(
signature_cipher: &str,
deobf: &Deobfuscator,
) -> Result<(String, BTreeMap<String, String>)> {
let params: HashMap<Cow<str>, Cow<str>> =
url::form_urlencoded::parse(signature_cipher.as_bytes()).collect();
// Parameters:
// `s`: Obfuscated signature
// `sp`: Signature parameter
// `url`: URL that is missing the signature parameter
let sig = some_or_bail!(params.get("s"), Err(anyhow!("no s param")));
let sp = some_or_bail!(params.get("sp"), Err(anyhow!("no sp param")));
let raw_url = some_or_bail!(params.get("url"), Err(anyhow!("no url param")));
let (url_base, mut url_params) = util::url_to_params(raw_url)?;
// println!("sig: {}", sig);
let deobf_sig = deobf.deobfuscate_sig(sig)?;
url_params.insert(sp.to_string(), deobf_sig);
Ok((url_base, url_params))
}
fn deobf_nsig(
url_params: &mut BTreeMap<String, String>,
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> Result<()> {
let nsig: String;
match url_params.get("n") {
Some(n) => {
nsig = if n.to_owned() == last_nsig[0] {
last_nsig[1].to_owned()
} else {
let nsig = deobf.deobfuscate_nsig(n)?;
last_nsig[0] = n.to_string();
last_nsig[1] = nsig.to_owned();
nsig
};
url_params.insert("n".to_owned(), nsig);
}
None => {}
};
Ok(())
}
fn map_url(
url: &Option<String>,
signature_cipher: &Option<String>,
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> MapResult<Option<(String, bool)>> {
let (url_base, mut url_params) = match url {
Some(url) => ok_or_bail!(
util::url_to_params(url),
MapResult {
c: None,
warnings: vec![format!("Could not parse url `{}`", url)]
}
),
None => match signature_cipher {
Some(signature_cipher) => match cipher_to_url_params(signature_cipher, deobf) {
Ok(res) => res,
Err(e) => {
return MapResult {
c: None,
warnings: vec![format!(
"Could not deobfuscate signatureCipher `{}`: {}",
signature_cipher, e
)],
};
}
},
None => {
return MapResult {
c: None,
warnings: vec!["stream contained neither url nor cipher".to_owned()],
}
}
},
};
let mut warnings = vec![];
let mut throttled = false;
deobf_nsig(&mut url_params, deobf, last_nsig).unwrap_or_else(|e| {
warnings.push(format!(
"Could not deobfuscate nsig (params: {:?}): {}",
url_params, e
));
throttled = true;
});
MapResult {
c: Some((
ok_or_bail!(
Url::parse_with_params(url_base.as_str(), url_params.iter()),
MapResult {
c: None,
warnings: vec![format!(
"url could not be joined. url: `{}` params: {:?}",
url_base, url_params
)],
}
)
.to_string(),
throttled,
)),
warnings,
}
}
fn map_video_stream(
f: player::Format,
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> MapResult<Option<VideoStream>> {
let (mtype, codecs) = some_or_bail!(
parse_mime(&f.mime_type),
MapResult {
c: None,
warnings: vec![format!(
"Invalid mime type `{}` in video format {:?}",
&f.mime_type, &f
)]
}
);
let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig);
match map_res.c {
Some((url, throttled)) => MapResult {
c: Some(VideoStream {
url,
itag: f.itag,
bitrate: f.bitrate,
average_bitrate: f.average_bitrate,
size: f.content_length,
index_range: f.index_range,
init_range: f.init_range,
width: some_or_bail!(
f.width,
MapResult {
c: None,
warnings: map_res.warnings
}
),
height: some_or_bail!(
f.height,
MapResult {
c: None,
warnings: map_res.warnings
}
),
fps: some_or_bail!(
f.fps,
MapResult {
c: None,
warnings: map_res.warnings
}
),
quality: some_or_bail!(
f.quality_label,
MapResult {
c: None,
warnings: map_res.warnings
}
),
hdr: f.color_info.unwrap_or_default().primaries
== player::Primaries::ColorPrimariesBt2020,
mime: f.mime_type.to_owned(),
format: some_or_bail!(
get_video_format(mtype),
MapResult {
c: None,
warnings: vec![format!("no valid format in video format")]
}
),
codec: get_video_codec(codecs),
throttled,
}),
warnings: map_res.warnings,
},
None => MapResult {
c: None,
warnings: map_res.warnings,
},
}
}
fn map_audio_stream(
f: player::Format,
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> MapResult<Option<AudioStream>> {
static LANG_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"^([a-z]{2})\."#).unwrap());
let (mtype, codecs) = some_or_bail!(
parse_mime(&f.mime_type),
MapResult {
c: None,
warnings: vec![format!(
"Invalid mime type `{}` in video format {:?}",
&f.mime_type, &f
)]
}
);
let map_res = map_url(&f.url, &f.signature_cipher, deobf, last_nsig);
match map_res.c {
Some((url, throttled)) => MapResult {
c: Some(AudioStream {
url,
itag: f.itag,
bitrate: f.bitrate,
average_bitrate: f.average_bitrate,
size: f.content_length,
index_range: f.index_range,
init_range: f.init_range,
mime: f.mime_type.to_owned(),
format: some_or_bail!(
get_audio_format(mtype),
MapResult {
c: None,
warnings: vec![format!("invalid format in audio format {}", f.itag)]
}
),
codec: get_audio_codec(codecs),
throttled,
track: match f.audio_track {
Some(t) => {
let lang = LANG_PATTERN
.captures(&t.id)
.ok()
.flatten()
.map(|m| m.get(1).unwrap().as_str().to_owned());
Some(AudioTrack {
id: t.id,
lang,
lang_name: t.display_name,
is_default: t.audio_is_default,
})
}
None => None,
},
}),
warnings: map_res.warnings,
},
None => MapResult {
c: None,
warnings: map_res.warnings,
},
}
}
fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> {
static PATTERN: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap());
let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), None);
Some((
captures.get(1).unwrap().as_str(),
captures
.get(2)
.unwrap()
.as_str()
.split(", ")
.collect::<Vec<&str>>(),
))
}
fn get_video_format(mtype: &str) -> Option<VideoFormat> {
match mtype {
"video/3gpp" => Some(VideoFormat::ThreeGp),
"video/mp4" => Some(VideoFormat::Mp4),
"video/webm" => Some(VideoFormat::Webm),
_ => None,
}
}
fn get_video_codec(codecs: Vec<&str>) -> VideoCodec {
for codec in codecs {
if codec.starts_with("avc1") {
return VideoCodec::Avc1;
} else if codec.starts_with("vp9") || codec.starts_with("vp09") {
return VideoCodec::Vp9;
} else if codec.starts_with("av01") {
return VideoCodec::Av01;
} else if codec.starts_with("mp4v") {
return VideoCodec::Mp4v;
}
}
VideoCodec::Unknown
}
fn get_audio_format(mtype: &str) -> Option<AudioFormat> {
match mtype {
"audio/mp4" => Some(AudioFormat::M4a),
"audio/webm" => Some(AudioFormat::Webm),
_ => None,
}
}
fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec {
for codec in codecs {
if codec.starts_with("mp4a") {
return AudioCodec::Mp4a;
} else if codec.starts_with("opus") {
return AudioCodec::Opus;
}
}
AudioCodec::Unknown
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader, path::Path};
use crate::{cache::DeobfData, client2::CLIENT_TYPES, report::TestFileReporter};
use super::*;
use rstest::rstest;
static DEOBFUSCATOR: Lazy<Deobfuscator> = Lazy::new(|| {
Deobfuscator::from(DeobfData {
js_url: "https://www.youtube.com/s/player/c8b8a173/player_ias.vflset/en_US/base.js".to_owned(),
sig_fn: "var oB={B4:function(a){a.reverse()},xm:function(a,b){a.splice(0,b)},dC:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};var Vva=function(a){a=a.split(\"\");oB.dC(a,42);oB.xm(a,3);oB.dC(a,48);oB.B4(a,68);return a.join(\"\")};function deobfuscate(a){return Vva(a);}".to_owned(),
nsig_fn: "Ska=function(a){var b=a.split(\"\"),c=[-1505243983,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(-e).reverse().forEach(function(f){d.unshift(f)})},\n-1692381986,function(d,e){e=(e%d.length+d.length)%d.length;var f=d[0];d[0]=d[e];d[e]=f},\n-262444939,\"unshift\",function(d){for(var e=d.length;e;)d.push(d.splice(--e,1)[0])},\n1201502951,-546377604,-504264123,-1978377336,1042456724,function(d,e){for(e=(e%d.length+d.length)%d.length;e--;)d.unshift(d.pop())},\n711986897,406699922,-1842537993,-1678108293,1803491779,1671716087,12778705,-718839990,null,null,-1617525823,342523552,-1338406651,-399705108,-696713950,b,function(d,e){e=(e%d.length+d.length)%d.length;d.splice(0,1,d.splice(e,1,d[0])[0])},\nfunction(d,e){e=(e%d.length+d.length)%d.length;d.splice(e,1)},\n-980602034,356396192,null,-1617525823,function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(\"\"))},\n-1029864222,-641353250,-1681901809,-1391247867,1707415199,-1957855835,b,function(){for(var d=64,e=[];++d-e.length-32;)switch(d){case 58:d=96;continue;case 91:d=44;break;case 65:d=47;continue;case 46:d=153;case 123:d-=58;default:e.push(String.fromCharCode(d))}return e},\n-1936558978,-1505243983,function(d){d.reverse()},\n1296889058,-1813915420,-943019300,function(d,e,f){var h=f.length;d.forEach(function(l,m,n){this.push(n[m]=f[(f.indexOf(l)-f.indexOf(this[m])+m+h--)%f.length])},e.split(\"\"))},\n\"join\",b,-2061642263];c[21]=c;c[22]=c;c[33]=c;try{c[3](c[33],c[9]),c[29](c[22],c[25]),c[29](c[22],c[19]),c[29](c[33],c[17]),c[29](c[21],c[2]),c[29](c[42],c[10]),c[1](c[52],c[40]),c[12](c[28],c[8]),c[29](c[21],c[45]),c[1](c[21],c[48]),c[44](c[26]),c[39](c[5],c[2]),c[31](c[53],c[16]),c[30](c[29],c[8]),c[51](c[29],c[6],c[44]()),c[4](c[43],c[1]),c[2](c[23],c[42]),c[2](c[0],c[46]),c[38](c[14],c[52]),c[32](c[5]),c[26](c[29],c[46]),c[26](c[5],c[13]),c[28](c[1],c[37]),c[26](c[31],c[13]),c[26](c[1],c[34]),\nc[46](c[1],c[32],c[40]()),c[26](c[50],c[44]),c[17](c[50],c[51]),c[0](c[3],c[24]),c[32](c[13]),c[43](c[3],c[51]),c[0](c[34],c[17]),c[16](c[45],c[53]),c[29](c[44],c[13]),c[42](c[1],c[50]),c[47](c[22],c[53]),c[37](c[22]),c[13](c[52],c[21]),c[6](c[43],c[34]),c[6](c[31],c[46])}catch(d){return\"enhanced_except_gZYB_un-_w8_\"+a}return b.join(\"\")};function deobfuscate(a){return Ska(a);}".to_owned(),
sts: "19201".to_owned(),
})
});
#[test_log::test(tokio::test)]
async fn download_response_testfiles() {
let tf_dir = Path::new("testfiles/player");
let video_id = "pPvd8UxmSbQ";
for client_type in CLIENT_TYPES {
let mut json_path = tf_dir.to_path_buf();
json_path.push(format!("{:?}_video.json", client_type).to_lowercase());
if json_path.exists() {
continue;
}
let reporter = TestFileReporter::new(json_path);
let rp = RustyPipe::new(None, Some(Box::new(reporter)), None).report(true);
rp.get_player(video_id, client_type).await.unwrap();
}
}
#[test_log::test(tokio::test)]
async fn download_model_testfiles() {
let tf_dir = Path::new("testfiles/player_model");
let rp = RustyPipe::default();
for (name, id) in [("multilanguage", "tVWWp1PqDus"), ("hdr", "LXb3EKWsInQ")] {
let mut json_path = tf_dir.to_path_buf();
json_path.push(format!("{}.json", name).to_lowercase());
if json_path.exists() {
continue;
}
let player_data = rp.get_player(id, ClientType::Desktop).await.unwrap();
let file = File::create(json_path).unwrap();
serde_json::to_writer_pretty(file, &player_data).unwrap();
}
}
#[rstest]
#[case::desktop("desktop")]
#[case::desktop_music("desktopmusic")]
#[case::tv_html5_embed("tvhtml5embed")]
#[case::android("android")]
#[case::ios("ios")]
fn t_map_player_data(#[case] name: &str) {
let filename = format!("testfiles/player/{}_video.json", name);
let json_path = Path::new(&filename);
let json_file = File::open(json_path).unwrap();
let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = resp
.map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBFUSCATOR))
.unwrap();
assert!(
map_res.warnings.is_empty(),
"deserialization/mapping warnings: {:?}",
map_res.warnings
);
let is_desktop = name == "desktop" || name == "desktopmusic";
insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), map_res.c, {
".info.publish_date" => insta::dynamic_redaction(move |value, _path| {
if is_desktop {
assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00"));
"2019-05-30T00:00:00"
} else {
assert_eq!(value, insta::internals::Content::None);
"~"
}
}),
});
}
/// Assert equality within 10% margin
fn assert_approx(left: f64, right: f64) {
if left != right {
let f = left / right;
assert!(
0.9 < f && f < 1.1,
"{} not within 10% margin of {}",
left,
right
);
}
}
#[rstest]
#[case::desktop(ClientType::Desktop)]
#[case::tv_html5_embed(ClientType::TvHtml5Embed)]
#[case::android(ClientType::Android)]
#[case::ios(ClientType::Ios)]
#[test_log::test(tokio::test)]
async fn t_get_player(#[case] client_type: ClientType) {
let rp = RustyPipe::default();
let player_data = rp.get_player("n4tK7LYFxI0", client_type).await.unwrap();
// dbg!(&player_data);
assert_eq!(player_data.info.id, "n4tK7LYFxI0");
assert_eq!(player_data.info.title, "Spektrem - Shine [NCS Release]");
if client_type == ClientType::DesktopMusic {
assert!(player_data.info.description.is_none());
} else {
assert!(player_data.info.description.unwrap().starts_with(
"NCS (NoCopyrightSounds): Empowering Creators through Copyright / Royalty Free Music"
));
}
assert_eq!(player_data.info.length, 259);
assert!(!player_data.info.thumbnails.is_empty());
assert_eq!(player_data.info.channel.id, "UC_aEa8K-EOJ3D6gOs7HcyNg");
assert_eq!(player_data.info.channel.name, "NoCopyrightSounds");
assert!(player_data.info.view_count > 146818808);
assert_eq!(player_data.info.keywords[0], "spektrem");
assert_eq!(player_data.info.is_live_content, false);
if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic {
assert!(player_data
.info
.publish_date
.unwrap()
.to_string()
.starts_with("2013-05-05 00:00:00"));
assert_eq!(player_data.info.category.unwrap(), "Music");
assert_eq!(player_data.info.is_family_safe.unwrap(), true);
}
if client_type == ClientType::Ios {
let video = player_data
.video_only_streams
.iter()
.find(|s| s.itag == 247)
.unwrap();
let audio = player_data
.audio_streams
.iter()
.find(|s| s.itag == 140)
.unwrap();
// Bitrates may change between requests
assert_approx(video.bitrate as f64, 1507068.0);
assert_eq!(video.average_bitrate, 1345149);
assert_eq!(video.size, 43553412);
assert_eq!(video.width, 1280);
assert_eq!(video.height, 720);
assert_eq!(video.fps, 30);
assert_eq!(video.quality, "720p");
assert_eq!(video.hdr, false);
assert_eq!(video.mime, "video/webm; codecs=\"vp09.00.31.08\"");
assert_eq!(video.format, VideoFormat::Webm);
assert_eq!(video.codec, VideoCodec::Vp9);
assert_approx(audio.bitrate as f64, 130685.0);
assert_eq!(audio.average_bitrate, 129496);
assert_eq!(audio.size, 4193863);
assert_eq!(audio.mime, "audio/mp4; codecs=\"mp4a.40.2\"");
assert_eq!(audio.format, AudioFormat::M4a);
assert_eq!(audio.codec, AudioCodec::Mp4a);
} else {
let video = player_data
.video_only_streams
.iter()
.find(|s| s.itag == 398)
.unwrap();
let audio = player_data
.audio_streams
.iter()
.find(|s| s.itag == 251)
.unwrap();
assert_approx(video.bitrate as f64, 1340829.0);
assert_approx(video.average_bitrate as f64, 1233444.0);
assert_approx(video.size as f64, 39936630.0);
assert_eq!(video.width, 1280);
assert_eq!(video.height, 720);
assert_eq!(video.fps, 30);
assert_eq!(video.quality, "720p");
assert_eq!(video.hdr, false);
assert_eq!(video.mime, "video/mp4; codecs=\"av01.0.05M.08\"");
assert_eq!(video.format, VideoFormat::Mp4);
assert_eq!(video.codec, VideoCodec::Av01);
assert_eq!(video.throttled, false);
assert_approx(audio.bitrate as f64, 142718.0);
assert_approx(audio.average_bitrate as f64, 130708.0);
assert_approx(audio.size as f64, 4232344.0);
assert_eq!(audio.mime, "audio/webm; codecs=\"opus\"");
assert_eq!(audio.format, AudioFormat::Webm);
assert_eq!(audio.codec, AudioCodec::Opus);
assert_eq!(audio.throttled, false);
}
assert!(player_data.expires_in_seconds > 10000);
}
#[test]
fn t_cipher_to_url() {
let signature_cipher = "s=w%3DAe%3DA6aDNQLkViKS7LOm9QtxZJHKwb53riq9qEFw-ecBWJCAiA%3DcEg0tn3dty9jEHszfzh4Ud__bg9CEHVx4ix-7dKsIPAhIQRw8JQ0qOA&sp=sig&url=https://rr5---sn-h0jelnez.googlevideo.com/videoplayback%3Fexpire%3D1659376413%26ei%3Dvb7nYvH5BMK8gAfBj7ToBQ%26ip%3D2003%253Ade%253Aaf06%253A6300%253Ac750%253A1b77%253Ac74a%253A80e3%26id%3Do-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2%26itag%3D251%26source%3Dyoutube%26requiressl%3Dyes%26mh%3DhH%26mm%3D31%252C26%26mn%3Dsn-h0jelnez%252Csn-4g5ednsl%26ms%3Dau%252Conr%26mv%3Dm%26mvi%3D5%26pl%3D37%26initcwndbps%3D1588750%26spc%3DlT-Khi831z8dTejFIRCvCEwx_6romtM%26vprv%3D1%26mime%3Daudio%252Fwebm%26ns%3Db_Mq_qlTFcSGlG9RpwpM9xQH%26gir%3Dyes%26clen%3D3781277%26dur%3D229.301%26lmt%3D1655510291473933%26mt%3D1659354538%26fvip%3D5%26keepalive%3Dyes%26fexp%3D24001373%252C24007246%26c%3DWEB%26rbqsm%3Dfr%26txp%3D4532434%26n%3Dd2g6G2hVqWIXxedQ%26sparams%3Dexpire%252Cei%252Cip%252Cid%252Citag%252Csource%252Crequiressl%252Cspc%252Cvprv%252Cmime%252Cns%252Cgir%252Cclen%252Cdur%252Clmt%26lsparams%3Dmh%252Cmm%252Cmn%252Cms%252Cmv%252Cmvi%252Cpl%252Cinitcwndbps%26lsig%3DAG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%253D%253D";
let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()];
let map_res = map_url(
&None,
&Some(signature_cipher.to_owned()),
&DEOBFUSCATOR,
&mut last_nsig,
);
let (url, throttled) = map_res.c.unwrap();
assert_eq!(url, "https://rr5---sn-h0jelnez.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=vb7nYvH5BMK8gAfBj7ToBQ&expire=1659376413&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AB_BABwrXZJN428ZwDxq5ScPn2AbcGODnRlTVhCQ3mj2&initcwndbps=1588750&ip=2003%3Ade%3Aaf06%3A6300%3Ac750%3A1b77%3Ac74a%3A80e3&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRQIgCKCGJ1iu4wlaGXy3jcJyU3inh9dr1FIfqYOZEG_MdmACIQCbungkQYFk7EhD6K2YvLaHFMjKOFWjw001_tLb0lPDtg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jelnez%2Csn-4g5ednsl&ms=au%2Conr&mt=1659354538&mv=m&mvi=5&n=XzXGSfGusw6OCQ&ns=b_Mq_qlTFcSGlG9RpwpM9xQH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhAPIsKd7-xi4xVHEC9gb__dU4hzfzsHEj9ytd3nt0gEceAiACJWBcw-wFEq9qir35bwKHJZxtQ9mOL7SKiVkLQNDa6A%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khi831z8dTejFIRCvCEwx_6romtM&txp=4532434&vprv=1");
assert_eq!(throttled, false);
assert!(
map_res.warnings.is_empty(),
"deserialization/mapping warnings: {:?}",
map_res.warnings
);
}
}

59
src/client2/playlist.rs
View file

@ -1,8 +1,9 @@
use anyhow::{anyhow, Result}; use anyhow::{anyhow, bail, Result};
use reqwest::Method; use reqwest::Method;
use serde::Serialize; use serde::Serialize;
use crate::{ use crate::{
deobfuscate::Deobfuscator,
model::{Channel, Language, Playlist, Thumbnail, Video}, model::{Channel, Language, Playlist, Thumbnail, Video},
serializer::text::{PageType, TextLink}, serializer::text::{PageType, TextLink},
timeago, util, timeago, util,
@ -39,6 +40,7 @@ impl RustyPipe {
"browse", "browse",
playlist_id, playlist_id,
&request_body, &request_body,
None,
) )
.await .await
} }
@ -60,6 +62,7 @@ impl RustyPipe {
"browse", "browse",
&playlist.id, &playlist.id,
&request_body, &request_body,
None,
) )
.await?; .await?;
@ -78,7 +81,12 @@ impl RustyPipe {
} }
impl MapResponse<Playlist> for response::Playlist { impl MapResponse<Playlist> for response::Playlist {
fn map_response(&self, lang: Language, id: &str) -> Result<super::MapResult<Playlist>> { fn map_response(
self,
id: &str,
lang: Language,
_deobf: Option<&Deobfuscator>,
) -> Result<MapResult<Playlist>> {
let video_items = &some_or_bail!( let video_items = &some_or_bail!(
some_or_bail!( some_or_bail!(
some_or_bail!( some_or_bail!(
@ -168,19 +176,15 @@ impl MapResponse<Playlist> for response::Playlist {
None => videos.len() as u32, None => videos.len() as u32,
}; };
let playlist_id = self.header.playlist_header_renderer.playlist_id.to_owned(); let playlist_id = self.header.playlist_header_renderer.playlist_id;
if playlist_id != id { if playlist_id != id {
return Err(anyhow!("got wrong playlist id {}, expected {}", playlist_id, id)); bail!("got wrong playlist id {}, expected {}", playlist_id, id);
} }
let name = self.header.playlist_header_renderer.title.to_owned(); let name = self.header.playlist_header_renderer.title;
let description = self let description = self.header.playlist_header_renderer.description_text;
.header
.playlist_header_renderer
.description_text
.to_owned();
let channel = match &self.header.playlist_header_renderer.owner_text { let channel = match self.header.playlist_header_renderer.owner_text {
Some(owner_text) => match owner_text { Some(owner_text) => match owner_text {
TextLink::Browse { TextLink::Browse {
text, text,
@ -188,8 +192,8 @@ impl MapResponse<Playlist> for response::Playlist {
browse_id, browse_id,
} => match page_type { } => match page_type {
PageType::Channel => Some(Channel { PageType::Channel => Some(Channel {
id: browse_id.to_owned(), id: browse_id,
name: text.to_owned(), name: text,
}), }),
_ => None, _ => None,
}, },
@ -198,6 +202,18 @@ impl MapResponse<Playlist> for response::Playlist {
None => None, None => None,
}; };
let mut warnings = video_items.warnings.to_owned();
let last_update = match &last_update_txt {
Some(textual_date) => {
let parsed = timeago::parse_textual_date_to_dt(lang, textual_date);
if parsed.is_none() {
warnings.push(format!("could not parse textual date `{}`", textual_date));
}
parsed
}
None => None,
};
Ok(MapResult { Ok(MapResult {
c: Playlist { c: Playlist {
id: playlist_id, id: playlist_id,
@ -208,22 +224,20 @@ impl MapResponse<Playlist> for response::Playlist {
thumbnails, thumbnails,
description, description,
channel, channel,
last_update: match &last_update_txt { last_update,
Some(textual_date) => timeago::parse_textual_date_to_dt(lang, textual_date),
None => None,
},
last_update_txt, last_update_txt,
}, },
warnings: video_items.warnings.to_owned(), warnings,
}) })
} }
} }
impl MapResponse<(Vec<Video>, Option<String>)> for response::PlaylistCont { impl MapResponse<(Vec<Video>, Option<String>)> for response::PlaylistCont {
fn map_response( fn map_response(
&self, self,
_lang: Language,
id: &str, id: &str,
_lang: Language,
_deobf: Option<&Deobfuscator>,
) -> Result<MapResult<(Vec<Video>, Option<String>)>> { ) -> Result<MapResult<(Vec<Video>, Option<String>)>> {
let action = some_or_bail!( let action = some_or_bail!(
self.on_response_received_actions self.on_response_received_actions
@ -237,7 +251,8 @@ impl MapResponse<(Vec<Video>, Option<String>)> for response::PlaylistCont {
warnings: action warnings: action
.append_continuation_items_action .append_continuation_items_action
.continuation_items .continuation_items
.warnings.to_owned(), .warnings
.to_owned(),
}) })
} }
} }
@ -383,7 +398,7 @@ mod tests {
let playlist: response::Playlist = let playlist: response::Playlist =
serde_json::from_reader(BufReader::new(json_file)).unwrap(); serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = playlist.map_response(Language::En, id).unwrap(); let map_res = playlist.map_response(id, Language::En, None).unwrap();
assert!( assert!(
map_res.warnings.is_empty(), map_res.warnings.is_empty(),

20
src/client2/response/mod.rs
View file

@ -214,3 +214,23 @@ pub struct MusicContinuation {
pub struct MusicContinuationData { pub struct MusicContinuationData {
pub continuation: String, pub continuation: String,
} }
impl Into<crate::model::Thumbnail> for Thumbnail {
fn into(self) -> crate::model::Thumbnail {
crate::model::Thumbnail {
url: self.url,
width: self.width,
height: self.height,
}
}
}
impl Into<Vec<crate::model::Thumbnail>> for Thumbnails {
fn into(self) -> Vec<crate::model::Thumbnail> {
let mut thumbnails = vec![];
for t in self.thumbnails {
thumbnails.push(t.into());
}
thumbnails
}
}

2
src/dictionary.rs
View file

@ -1,7 +1,7 @@
// This file is automatically generated. DO NOT EDIT. // This file is automatically generated. DO NOT EDIT.
use crate::{ use crate::{
model::Language, model::Language,
timeago::{TaToken, TimeUnit, DateCmp}, timeago::{DateCmp, TaToken, TimeUnit},
}; };
pub struct Entry { pub struct Entry {

8
src/lib.rs
View file

@ -3,8 +3,8 @@
#[macro_use] #[macro_use]
mod macros; mod macros;
#[cfg(test)] // #[cfg(test)]
mod codegen; // mod codegen;
mod deobfuscate; mod deobfuscate;
mod dictionary; mod dictionary;
@ -12,9 +12,9 @@ mod serializer;
mod timeago; mod timeago;
mod util; mod util;
pub mod client; // pub mod client;
pub mod cache;
pub mod client2; pub mod client2;
pub mod download; pub mod download;
pub mod model; pub mod model;
pub mod cache;
pub mod report; pub mod report;

2
src/report.rs
View file

@ -30,8 +30,6 @@ pub struct Report {
// pub deobf_data: Option<DeobfData>, // pub deobf_data: Option<DeobfData>,
/// HTTP request data /// HTTP request data
pub http_request: HTTPRequest, pub http_request: HTTPRequest,
/// Debug print of deserialized response object
pub deserialized: Option<String>,
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]

2
src/serializer/mod.rs
View file

@ -1,5 +1,5 @@
mod vec_log_err;
pub mod range; pub mod range;
pub mod text; pub mod text;
mod vec_log_err;
pub use vec_log_err::VecLogError; pub use vec_log_err::VecLogError;