Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add custom error types, remove anyhow

Browse source
This commit is contained in:
ThetaDev 2022-10-08 14:30:09 +02:00
parent 1297bcb641
commit a3e3269fb3
16 changed files with 385 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Cargo.toml
View file

@ -13,7 +13,7 @@ include = ["/src", "README.md", "LICENSE", "!snapshots"]
members = [".", "codegen", "cli"]
[features]
default = ["default-tls"]
default = ["default-tls", "rss"]
all = ["rss", "html"]
rss = ["quick-xml"]
@ -26,10 +26,10 @@ rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
[dependencies]
# quick-js = "0.4.1"
quick-js = { path = "../quickjs-rs" }
quick-js = { path = "../quickjs-rs", default-features = false }
once_cell = "1.12.0"
fancy-regex = "0.10.0"
anyhow = "1.0"
thiserror = "1.0.36"
url = "2.2.2"
log = "0.4.17"
reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]}
@ -38,7 +38,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.82"
serde_with = {version = "2.0.0", features = ["json"] }
rand = "0.8.5"
chrono = {version = "0.4.19", features = ["serde"]}
chrono = {version = "0.4.19", default-features = false, features = ["clock", "serde"]}
chronoutil = "0.2.3"
futures = "0.3.21"
indicatif = "0.17.0"

2
src/cache.rs
View file

@ -7,7 +7,7 @@ use std::{
use log::error;
pub trait CacheStorage {
pub trait CacheStorage: Sync + Send {
fn write(&self, data: &str);
fn read(&self) -> Option<String>;
}

37
src/client/channel.rs
View file

@ -1,9 +1,9 @@
use anyhow::{anyhow, bail, Result};
use chrono::TimeZone;
use serde::Serialize;
use url::Url;
use crate::{
error::{Error, ExtractionError},
model::{
Channel, ChannelInfo, ChannelOrder, ChannelPlaylist, ChannelVideo, Language, Paginator,
},
@ -43,7 +43,7 @@ impl RustyPipeQuery {
pub async fn channel_videos(
&self,
channel_id: &str,
) -> Result<Channel<Paginator<ChannelVideo>>> {
) -> Result<Channel<Paginator<ChannelVideo>>, Error> {
self.channel_videos_ordered(channel_id, ChannelOrder::default())
.await
}
@ -52,7 +52,7 @@ impl RustyPipeQuery {
&self,
channel_id: &str,
order: ChannelOrder,
) -> Result<Channel<Paginator<ChannelVideo>>> {
) -> Result<Channel<Paginator<ChannelVideo>>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QChannel {
context,
@ -77,7 +77,7 @@ impl RustyPipeQuery {
pub async fn channel_videos_continuation(
&self,
ctoken: &str,
) -> Result<Paginator<ChannelVideo>> {
) -> Result<Paginator<ChannelVideo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QContinuation {
context,
@ -97,7 +97,7 @@ impl RustyPipeQuery {
pub async fn channel_playlists(
&self,
channel_id: &str,
) -> Result<Channel<Paginator<ChannelPlaylist>>> {
) -> Result<Channel<Paginator<ChannelPlaylist>>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QChannel {
context,
@ -118,7 +118,7 @@ impl RustyPipeQuery {
pub async fn channel_playlists_continuation(
&self,
ctoken: &str,
) -> Result<Paginator<ChannelPlaylist>> {
) -> Result<Paginator<ChannelPlaylist>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QContinuation {
context,
@ -135,7 +135,7 @@ impl RustyPipeQuery {
.await
}
pub async fn channel_info(&self, channel_id: &str) -> Result<Channel<ChannelInfo>> {
pub async fn channel_info(&self, channel_id: &str) -> Result<Channel<ChannelInfo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QChannel {
context,
@ -160,7 +160,7 @@ impl MapResponse<Channel<Paginator<ChannelVideo>>> for response::Channel {
id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<Paginator<ChannelVideo>>>> {
) -> Result<MapResult<Channel<Paginator<ChannelVideo>>>, ExtractionError> {
let content = map_channel_content(self.contents, id);
let mut warnings = content.warnings;
let grid = match content.c {
@ -191,7 +191,7 @@ impl MapResponse<Channel<Paginator<ChannelPlaylist>>> for response::Channel {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<Paginator<ChannelPlaylist>>>> {
) -> Result<MapResult<Channel<Paginator<ChannelPlaylist>>>, ExtractionError> {
let content = map_channel_content(self.contents, id);
let mut warnings = content.warnings;
let grid = match content.c {
@ -222,7 +222,7 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<ChannelInfo>>> {
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
let content = map_channel_content(self.contents, id);
let mut warnings = content.warnings;
let meta = match content.c {
@ -278,11 +278,11 @@ impl MapResponse<Paginator<ChannelVideo>> for response::ChannelCont {
_id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<ChannelVideo>>> {
) -> Result<MapResult<Paginator<ChannelVideo>>, ExtractionError> {
let mut actions = self.on_response_received_actions;
let res = some_or_bail!(
actions.try_swap_remove(0),
Err(anyhow!("no received action"))
Err(ExtractionError::InvalidData("no received action".into()))
)
.append_continuation_items_action
.continuation_items;
@ -297,11 +297,11 @@ impl MapResponse<Paginator<ChannelPlaylist>> for response::ChannelCont {
_id: &str,
_lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<ChannelPlaylist>>> {
) -> Result<MapResult<Paginator<ChannelPlaylist>>, ExtractionError> {
let mut actions = self.on_response_received_actions;
let res = some_or_bail!(
actions.try_swap_remove(0),
Err(anyhow!("no received action"))
Err(ExtractionError::InvalidData("no received action".into()))
)
.append_continuation_items_action
.continuation_items;
@ -423,15 +423,14 @@ fn map_channel<T>(
content: T,
id: &str,
lang: Language,
) -> Result<Channel<T>> {
) -> Result<Channel<T>, ExtractionError> {
let header = header.c4_tabbed_header_renderer;
if header.channel_id != id {
bail!(
return Err(ExtractionError::WrongResult(format!(
"got wrong channel id {}, expected {}",
header.channel_id,
id
);
header.channel_id, id
)));
}
Ok(Channel {

15
src/client/channel_rss.rs
View file

@ -1,13 +1,15 @@
use std::collections::BTreeMap;
use anyhow::Result;
use crate::{model::ChannelRss, report::Report};
use crate::{
error::{Error, ExtractionError},
model::ChannelRss,
report::Report,
};
use super::{response, RustyPipeQuery};
impl RustyPipeQuery {
pub async fn channel_rss(&self, channel_id: &str) -> Result<ChannelRss> {
pub async fn channel_rss(&self, channel_id: &str) -> Result<ChannelRss, Error> {
let url = format!(
"https://www.youtube.com/feeds/videos.xml?channel_id={}",
channel_id
@ -41,7 +43,10 @@ impl RustyPipeQuery {
reporter.report(&report);
}
Err(e.into())
Err(ExtractionError::InvalidData(
format!("could not deserialize xml: {}", e).into(),
)
.into())
}
}
}

73
src/client/mod.rs
View file

@ -14,7 +14,6 @@ mod channel_rss;
use std::fmt::Debug;
use std::sync::Arc;
use anyhow::{anyhow, bail, Context, Result};
use chrono::{DateTime, Duration, Utc};
use fancy_regex::Regex;
use log::{debug, error, warn};
@ -27,6 +26,7 @@ use tokio::sync::RwLock;
use crate::{
cache::{CacheStorage, FileStorage},
deobfuscate::{DeobfData, Deobfuscator},
error::{Error, ExtractionError, Result},
model::{Country, Language},
report::{FileReporter, Level, Report, Reporter},
serializer::MapResult,
@ -166,8 +166,8 @@ pub struct RustyPipe {
struct RustyPipeRef {
http: Client,
storage: Option<Box<dyn CacheStorage + Sync + Send>>,
reporter: Option<Box<dyn Reporter + Sync + Send>>,
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
n_retries: u32,
consent_cookie: String,
cache: CacheHolder,
@ -183,8 +183,8 @@ struct RustyPipeOpts {
}
pub struct RustyPipeBuilder {
storage: Option<Box<dyn CacheStorage + Sync + Send>>,
reporter: Option<Box<dyn Reporter + Sync + Send>>,
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
n_retries: u32,
user_agent: String,
default_opts: RustyPipeOpts,
@ -452,8 +452,11 @@ impl RustyPipe {
}
/// Execute the given http request.
async fn http_request(&self, request: Request) -> Result<Response, reqwest::Error> {
let mut last_res: Option<Result<Response, reqwest::Error>> = None;
async fn http_request(
&self,
request: Request,
) -> core::result::Result<Response, reqwest::Error> {
let mut last_res = None;
for n in 0..self.inner.n_retries {
let res = self.inner.http.execute(request.try_clone().unwrap()).await;
let emsg = match &res {
@ -509,11 +512,13 @@ impl RustyPipe {
.build()
.unwrap(),
)
.await
.context("Failed to download sw.js")?;
.await?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
.ok_or_else(|| anyhow!("Could not find desktop client version in sw.js"))
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from(
ExtractionError::InvalidData(
"Could not find desktop client version in sw.js".into(),
),
))
};
let from_html = async {
@ -525,11 +530,13 @@ impl RustyPipe {
.build()
.unwrap(),
)
.await
.context("Failed to get YT Desktop page")?;
.await?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1)
.ok_or_else(|| anyhow!("Could not find desktop client version on html page"))
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from(
ExtractionError::InvalidData(
"Could not find desktop client version in sw.js".into(),
),
))
};
match from_swjs.await {
@ -552,11 +559,11 @@ impl RustyPipe {
.build()
.unwrap(),
)
.await
.context("Failed to download sw.js")?;
.await?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
.ok_or_else(|| anyhow!("Could not find desktop client version in sw.js"))
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from(
ExtractionError::InvalidData("Could not find music client version in sw.js".into()),
))
};
let from_html = async {
@ -568,11 +575,13 @@ impl RustyPipe {
.build()
.unwrap(),
)
.await
.context("Failed to get YT Desktop page")?;
.await?;
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1)
.ok_or_else(|| anyhow!("Could not find desktop client version on html page"))
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from(
ExtractionError::InvalidData(
"Could not find music client version on html page".into(),
),
))
};
match from_swjs.await {
@ -971,7 +980,7 @@ impl RustyPipeQuery {
};
if status.is_client_error() || status.is_server_error() {
let e = anyhow!("Server responded with error code {}", status);
let e = Error::HttpStatus(status.into());
create_report(Level::ERR, Some(e.to_string()), vec![]);
return Err(e);
}
@ -982,12 +991,12 @@ impl RustyPipeQuery {
if !mapres.warnings.is_empty() {
create_report(
Level::WRN,
Some("Warnings during deserialization/mapping".to_owned()),
Some(ExtractionError::Warnings.to_string()),
mapres.warnings,
);
if self.opts.strict {
bail!("Warnings during deserialization/mapping");
return Err(Error::Extraction(ExtractionError::Warnings));
}
} else if self.opts.report {
create_report(Level::DBG, None, vec![]);
@ -995,15 +1004,13 @@ impl RustyPipeQuery {
Ok(mapres.c)
}
Err(e) => {
let emsg = "Could not map reponse";
create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
Err(e).context(emsg)
create_report(Level::ERR, Some(e.to_string()), Vec::new());
Err(e.into())
}
},
Err(e) => {
let emsg = "Could not deserialize response";
create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
Err(e).context(emsg)
create_report(Level::ERR, Some(e.to_string()), Vec::new());
Err(Error::from(ExtractionError::from(e)))
}
}
}
@ -1058,7 +1065,7 @@ trait MapResponse<T> {
id: &str,
lang: Language,
deobf: Option<&Deobfuscator>,
) -> Result<MapResult<T>>;
) -> core::result::Result<MapResult<T>, crate::error::ExtractionError>;
}
#[cfg(test)]

2
src/client/pagination.rs
View file

@ -1,4 +1,4 @@
use anyhow::Result;
use crate::error::Result;
use crate::model::{
ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo,

58
src/client/player.rs
View file

@ -3,7 +3,6 @@ use std::{
collections::{BTreeMap, HashMap},
};
use anyhow::{anyhow, bail, Result};
use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
use fancy_regex::Regex;
use once_cell::sync::Lazy;
@ -12,6 +11,7 @@ use url::Url;
use crate::{
deobfuscate::Deobfuscator,
error::{DeobfError, Error, ExtractionError},
model::{
AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Language, Subtitle,
VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream,
@ -58,7 +58,11 @@ struct QContentPlaybackContext {
}
impl RustyPipeQuery {
pub async fn player(self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
pub async fn player(
self,
video_id: &str,
client_type: ClientType,
) -> Result<VideoPlayer, Error> {
let q1 = self.clone();
let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await });
let q2 = self.client.clone();
@ -111,7 +115,7 @@ impl MapResponse<VideoPlayer> for response::Player {
id: &str,
_lang: Language,
deobf: Option<&Deobfuscator>,
) -> Result<super::MapResult<VideoPlayer>> {
) -> Result<super::MapResult<VideoPlayer>, ExtractionError> {
let deobf = deobf.unwrap();
let mut warnings = vec![];
@ -121,26 +125,36 @@ impl MapResponse<VideoPlayer> for response::Player {
live_streamability.is_some()
}
response::player::PlayabilityStatus::Unplayable { reason } => {
bail!("Video is unplayable. Reason: {}", reason)
return Err(ExtractionError::VideoUnavailable("DRM/Geoblock", reason))
}
response::player::PlayabilityStatus::LoginRequired { reason } => {
bail!("Playback requires login. Reason: {}", reason)
// reason: "Sign in to confirm your age"
if reason.split_whitespace().any(|word| word == "age") {
return Err(ExtractionError::VideoAgeRestricted);
}
return Err(ExtractionError::VideoUnavailable("private video", reason));
}
response::player::PlayabilityStatus::LiveStreamOffline { reason } => {
bail!("Livestream is offline. Reason: {}", reason)
return Err(ExtractionError::VideoUnavailable(
"offline livestream",
reason,
))
}
response::player::PlayabilityStatus::Error { reason } => {
bail!("Video was deleted. Reason: {}", reason)
return Err(ExtractionError::VideoUnavailable(
"deletion/censorship",
reason,
))
}
};
let mut streaming_data = some_or_bail!(
self.streaming_data,
Err(anyhow!("No streaming data was returned"))
Err(ExtractionError::InvalidData("no streaming data".into()))
);
let video_details = some_or_bail!(
self.video_details,
Err(anyhow!("No video details were returned"))
Err(ExtractionError::InvalidData("no video details".into()))
);
let microformat = self.microformat.map(|m| m.player_microformat_renderer);
let (publish_date, category, tags, is_family_safe) =
@ -159,11 +173,10 @@ impl MapResponse<VideoPlayer> for response::Player {
});
if video_details.video_id != id {
bail!(
"got wrong video id {}, expected {}",
video_details.video_id,
id
);
return Err(ExtractionError::WrongResult(format!(
"video id {}, expected {}",
video_details.video_id, id
)));
}
let video_info = VideoPlayerDetails {
@ -272,7 +285,7 @@ impl MapResponse<VideoPlayer> for response::Player {
fn cipher_to_url_params(
signature_cipher: &str,
deobf: &Deobfuscator,
) -> Result<(String, BTreeMap<String, String>)> {
) -> Result<(String, BTreeMap<String, String>), DeobfError> {
let params: HashMap<Cow<str>, Cow<str>> =
url::form_urlencoded::parse(signature_cipher.as_bytes()).collect();
@ -281,12 +294,15 @@ fn cipher_to_url_params(
// `sp`: Signature parameter
// `url`: URL that is missing the signature parameter
let sig = some_or_bail!(params.get("s"), Err(anyhow!("no s param")));
let sp = some_or_bail!(params.get("sp"), Err(anyhow!("no sp param")));
let raw_url = some_or_bail!(params.get("url"), Err(anyhow!("no url param")));
let (url_base, mut url_params) = util::url_to_params(raw_url)?;
let sig = some_or_bail!(params.get("s"), Err(DeobfError::Extraction("s param")));
let sp = some_or_bail!(params.get("sp"), Err(DeobfError::Extraction("sp param")));
let raw_url = some_or_bail!(
params.get("url"),
Err(DeobfError::Extraction("no url param"))
);
let (url_base, mut url_params) =
util::url_to_params(raw_url).or(Err(DeobfError::Extraction("url params")))?;
// println!("sig: {}", sig);
let deobf_sig = deobf.deobfuscate_sig(sig)?;
url_params.insert(sp.to_string(), deobf_sig);
@ -297,7 +313,7 @@ fn deobf_nsig(
url_params: &mut BTreeMap<String, String>,
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> Result<()> {
) -> Result<(), DeobfError> {
let nsig: String;
if let Some(n) = url_params.get("n") {
nsig = if n == &last_nsig[0] {

40
src/client/playlist.rs
View file

@ -1,10 +1,10 @@
use std::convert::TryFrom;
use anyhow::{anyhow, bail, Result};
use serde::Serialize;
use crate::{
deobfuscate::Deobfuscator,
error::{Error, ExtractionError},
model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo},
timeago,
util::{self, TryRemove},
@ -22,7 +22,7 @@ struct QPlaylist {
}
impl RustyPipeQuery {
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist> {
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QPlaylist {
context,
@ -39,7 +39,10 @@ impl RustyPipeQuery {
.await
}
pub async fn playlist_continuation(self, ctoken: &str) -> Result<Paginator<PlaylistVideo>> {
pub async fn playlist_continuation(
self,
ctoken: &str,
) -> Result<Paginator<PlaylistVideo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QContinuation {
context,
@ -63,26 +66,32 @@ impl MapResponse<Playlist> for response::Playlist {
id: &str,
lang: Language,
_deobf: Option<&Deobfuscator>,
) -> Result<MapResult<Playlist>> {
) -> Result<MapResult<Playlist>, ExtractionError> {
// TODO: think about a deserializer that deserializes only first list item
let mut tcbr_contents = self.contents.two_column_browse_results_renderer.contents;
let video_items = some_or_bail!(
some_or_bail!(
some_or_bail!(
tcbr_contents.try_swap_remove(0),
Err(anyhow!("twoColumnBrowseResultsRenderer empty"))
Err(ExtractionError::InvalidData(
"twoColumnBrowseResultsRenderer empty".into()
))
)
.tab_renderer
.content
.section_list_renderer
.contents
.try_swap_remove(0),
Err(anyhow!("sectionListRenderer empty"))
Err(ExtractionError::InvalidData(
"sectionListRenderer empty".into()
))
)
.item_section_renderer
.contents
.try_swap_remove(0),
Err(anyhow!("itemSectionRenderer empty"))
Err(ExtractionError::InvalidData(
"itemSectionRenderer empty".into()
))
)
.playlist_video_list_renderer
.contents;
@ -94,7 +103,7 @@ impl MapResponse<Playlist> for response::Playlist {
let mut sidebar_items = sidebar.playlist_sidebar_renderer.items;
let mut primary = some_or_bail!(
sidebar_items.try_swap_remove(0),
Err(anyhow!("no primary sidebar"))
Err(ExtractionError::InvalidData("no primary sidebar".into()))
);
(
@ -112,7 +121,7 @@ impl MapResponse<Playlist> for response::Playlist {
None => {
let header_banner = some_or_bail!(
self.header.playlist_header_renderer.playlist_header_banner,
Err(anyhow!("no thumbnail found"))
Err(ExtractionError::InvalidData("no thumbnail found".into()))
);
let mut byline = self.header.playlist_header_renderer.byline;
@ -131,7 +140,7 @@ impl MapResponse<Playlist> for response::Playlist {
Some(_) => {
ok_or_bail!(
util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text),
Err(anyhow!("no video count"))
Err(ExtractionError::InvalidData("no video count".into()))
)
}
None => videos.len() as u32,
@ -139,7 +148,10 @@ impl MapResponse<Playlist> for response::Playlist {
let playlist_id = self.header.playlist_header_renderer.playlist_id;
if playlist_id != id {
bail!("got wrong playlist id {}, expected {}", playlist_id, id);
return Err(ExtractionError::WrongResult(format!(
"got wrong playlist id {}, expected {}",
playlist_id, id
)));
}
let name = self.header.playlist_header_renderer.title;
@ -178,11 +190,13 @@ impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
_id: &str,
_lang: Language,
_deobf: Option<&Deobfuscator>,
) -> Result<MapResult<Paginator<PlaylistVideo>>> {
) -> Result<MapResult<Paginator<PlaylistVideo>>, ExtractionError> {
let mut actions = self.on_response_received_actions;
let action = some_or_bail!(
actions.try_swap_remove(0),
Err(anyhow!("no continuation action"))
Err(ExtractionError::InvalidData(
"no continuation action".into()
))
);
let (items, ctoken) =

46
src/client/video_details.rs
View file

@ -1,9 +1,9 @@
use std::convert::TryFrom;
use anyhow::{anyhow, bail, Result};
use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{
ChannelId, ChannelTag, Chapter, Comment, Language, Paginator, RecommendedVideo,
VideoDetails,
@ -30,7 +30,7 @@ struct QVideo {
}
impl RustyPipeQuery {
pub async fn video_details(self, video_id: &str) -> Result<VideoDetails> {
pub async fn video_details(self, video_id: &str) -> Result<VideoDetails, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QVideo {
context,
@ -49,7 +49,10 @@ impl RustyPipeQuery {
.await
}
pub async fn video_recommendations(self, ctoken: &str) -> Result<Paginator<RecommendedVideo>> {
pub async fn video_recommendations(
self,
ctoken: &str,
) -> Result<Paginator<RecommendedVideo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QContinuation {
context,
@ -66,7 +69,7 @@ impl RustyPipeQuery {
.await
}
pub async fn video_comments(self, ctoken: &str) -> Result<Paginator<Comment>> {
pub async fn video_comments(self, ctoken: &str) -> Result<Paginator<Comment>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QContinuation {
context,
@ -90,12 +93,15 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<VideoDetails>> {
) -> Result<MapResult<VideoDetails>, ExtractionError> {
let mut warnings = Vec::new();
let video_id = self.current_video_endpoint.watch_endpoint.video_id;
if id != video_id {
bail!("got wrong playlist id {}, expected {}", video_id, id);
return Err(ExtractionError::WrongResult(format!(
"got wrong playlist id {}, expected {}",
video_id, id
)));
}
let mut primary_results = self
@ -167,7 +173,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
view_count.video_view_count_renderer.is_live,
)
}
_ => bail!("could not find primary_info"),
_ => {
return Err(ExtractionError::InvalidData(
"could not find primary_info".into(),
))
}
};
let comment_count = comment_count_section.and_then(|s| {
@ -214,7 +224,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
(owner.video_owner_renderer, desc, is_ccommons)
}
_ => bail!("could not find secondary_info"),
_ => {
return Err(ExtractionError::InvalidData(
"could not find secondary_info".into(),
))
}
};
let (channel_id, channel_name) = match owner.title {
@ -224,9 +238,13 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
browse_id,
} => match page_type {
crate::serializer::text::PageType::Channel => (browse_id, text),
_ => bail!("invalid channel link type"),
_ => {
return Err(ExtractionError::InvalidData(
"invalid channel link type".into(),
))
}
},
_ => bail!("invalid channel link"),
_ => return Err(ExtractionError::InvalidData("invalid channel link".into())),
};
let recommended = self
@ -324,11 +342,13 @@ impl MapResponse<Paginator<RecommendedVideo>> for response::VideoRecommendations
_id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<RecommendedVideo>>> {
) -> Result<MapResult<Paginator<RecommendedVideo>>, ExtractionError> {
let mut endpoints = self.on_response_received_endpoints;
let cont = some_or_bail!(
endpoints.try_swap_remove(0),
Err(anyhow!("no continuation endpoint"))
Err(ExtractionError::InvalidData(
"no continuation endpoint".into()
))
);
Ok(map_recommendations(
@ -344,7 +364,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
_id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<Comment>>> {
) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
let mut warnings = self.on_response_received_endpoints.warnings;
let mut comments = Vec::new();

82
src/deobfuscate.rs
View file

@ -1,4 +1,3 @@
use anyhow::{anyhow, bail, Context, Result};
use fancy_regex::Regex;
use log::debug;
use once_cell::sync::Lazy;
@ -6,7 +5,9 @@ use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::result::Result::Ok;
use crate::util;
use crate::{error::DeobfError, util};
type Result<T> = core::result::Result<T, DeobfError>;
pub struct Deobfuscator {
data: DeobfData,
@ -22,13 +23,8 @@ pub struct DeobfData {
impl Deobfuscator {
pub async fn new(http: Client) -> Result<Self> {
let js_url = get_player_js_url(&http)
.await
.context("Failed to retrieve player.js URL")?;
let player_js = get_response(&http, &js_url)
.await
.context("Failed to download player.js")?;
let js_url = get_player_js_url(&http).await?;
let player_js = get_response(&http, &js_url).await?;
debug!("Downloaded player.js from {}", js_url);
@ -84,7 +80,7 @@ fn get_sig_fn_name(player_js: &str) -> Result<String> {
});
util::get_cg_from_regexes(FUNCTION_PATTERNS.iter(), player_js, 1)
.ok_or_else(|| anyhow!("could not find deobf function name"))
.ok_or(DeobfError::Extraction("deobf function name"))
}
fn caller_function(fn_name: &str) -> String {
@ -98,13 +94,13 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
"(".to_owned() + &dfunc_name.replace('$', "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
let function_pattern = ok_or_bail!(
Regex::new(&function_pattern_str),
Err(anyhow!("could not parse function pattern regex"))
Err(DeobfError::Other("could not parse function pattern regex"))
);
let deobfuscate_function = "var ".to_owned()
+ some_or_bail!(
function_pattern.captures(player_js).ok().flatten(),
Err(anyhow!("could not find deobf function"))
Err(DeobfError::Extraction("deobf function"))
)
.get(1)
.unwrap()
@ -118,7 +114,7 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
.captures(&deobfuscate_function)
.ok()
.flatten(),
Err(anyhow!("could not find helper object name"))
Err(DeobfError::Extraction("helper object name"))
)
.get(1)
.unwrap()
@ -128,12 +124,12 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
"(var ".to_owned() + &helper_object_name.replace('$', "\\$") + "=\\{.+?\\}\\};)";
let helper_pattern = ok_or_bail!(
Regex::new(&helper_pattern_str),
Err(anyhow!("could not parse helper pattern regex"))
Err(DeobfError::Other("could not parse helper pattern regex"))
);
let player_js_nonl = player_js.replace('\n', "");
let helper_object = some_or_bail!(
helper_pattern.captures(&player_js_nonl).ok().flatten(),
Err(anyhow!("could not find helper object"))
Err(DeobfError::Extraction("helper object"))
)
.get(1)
.unwrap()
@ -143,14 +139,15 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
}
fn deobfuscate_sig(sig: &str, sig_fn: &str) -> Result<String> {
let context = quick_js::Context::new()?;
let context =
quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?;
context.eval(sig_fn)?;
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
match res.as_str() {
Some(res) => Ok(res.to_owned()),
None => bail!("deobfuscation func returned null"),
}
res.as_str().map_or(
Err(DeobfError::Other("sig deobfuscation func returned null")),
|res| Ok(res.to_owned()),
)
}
fn get_nsig_fn_name(player_js: &str) -> Result<String> {
@ -161,7 +158,7 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
let fname_match = some_or_bail!(
FUNCTION_NAME_PATTERN.captures(player_js).ok().flatten(),
Err(anyhow!("could not find n_deobf function"))
Err(DeobfError::Extraction("n_deobf function"))
);
let function_name = fname_match.get(1).unwrap().as_str();
@ -170,25 +167,29 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
return Ok(function_name.to_owned());
}
let array_num = fname_match.get(2).unwrap().as_str().parse::<u32>()?;
let array_num = fname_match
.get(2)
.unwrap()
.as_str()
.parse::<usize>()
.or(Err(DeobfError::Other("could not parse array_num")))?;
let array_pattern_str =
"var ".to_owned() + &fancy_regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];";
let array_pattern = Regex::new(&array_pattern_str)?;
let array_pattern = Regex::new(&array_pattern_str).or(Err(DeobfError::Other(
"could not parse helper pattern regex",
)))?;
let array_str = some_or_bail!(
array_pattern.captures(player_js).ok().flatten(),
Err(anyhow!("could not find n_deobf array_str"))
Err(DeobfError::Extraction("n_deobf array_str"))
)
.get(1)
.unwrap()
.as_str();
let mut names = array_str.split(',');
let name = some_or_bail!(
names.nth(array_num.try_into()?),
Err(anyhow!(
"could not get {}th item from {}",
array_num,
array_str
))
names.nth(array_num),
Err(DeobfError::Extraction("n_deobf function name"))
);
Ok(name.to_owned())
}
@ -239,7 +240,7 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String> {
}
if state != 3 {
return Err(anyhow!("could not extract js fn"));
return Err(DeobfError::Extraction("javascript function"));
}
Ok(js[start..end].to_owned())
@ -255,14 +256,15 @@ fn get_nsig_fn(player_js: &str) -> Result<String> {
}
fn deobfuscate_nsig(sig: &str, nsig_fn: &str) -> Result<String> {
let context = quick_js::Context::new()?;
let context =
quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?;
context.eval(nsig_fn)?;
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
match res.as_str() {
Some(res) => Ok(res.to_owned()),
None => bail!("deobfuscation func returned null"),
}
res.as_str().map_or(
Err(DeobfError::Other("nsig deobfuscation func returned null")),
|res| Ok(res.to_owned()),
)
}
async fn get_player_js_url(http: &Client) -> Result<String> {
@ -278,8 +280,8 @@ async fn get_player_js_url(http: &Client) -> Result<String> {
.unwrap()
});
let player_hash = some_or_bail!(
PLAYER_HASH_PATTERN.captures(&text)?,
Err(anyhow!("could not find player hash"))
PLAYER_HASH_PATTERN.captures(&text).ok().flatten(),
Err(DeobfError::Extraction("player hash"))
)
.get(1)
.unwrap()
@ -301,8 +303,8 @@ fn get_sts(player_js: &str) -> Result<String> {
Lazy::new(|| Regex::new("signatureTimestamp[=:](\\d+)").unwrap());
Ok(some_or_bail!(
STS_PATTERN.captures(player_js)?,
Err(anyhow!("could not find sts"))
STS_PATTERN.captures(player_js).ok().flatten(),
Err(DeobfError::Extraction("sts"))
)
.get(1)
.unwrap()

84
src/download.rs
View file

@ -2,7 +2,6 @@
use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf};
use anyhow::{anyhow, bail, Result};
use fancy_regex::Regex;
use futures::stream::{self, StreamExt};
use indicatif::ProgressBar;
@ -17,10 +16,13 @@ use tokio::{
};
use crate::{
error::DownloadError,
model::{stream_filter::Filter, AudioCodec, FileFormat, VideoCodec, VideoPlayer},
util,
};
type Result<T> = core::result::Result<T, DownloadError>;
const CHUNK_SIZE_MIN: u64 = 9000000;
const CHUNK_SIZE_MAX: u64 = 10000000;
@ -44,15 +46,32 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
let captures = some_or_bail!(
PATTERN.captures(cr_header).ok().flatten(),
Err(anyhow!(
"Content-Range header '{}' does not match pattern.",
cr_header
Err(DownloadError::Progressive(
format!(
"Content-Range header '{}' does not match pattern",
cr_header
)
.into()
))
);
Ok((
captures.get(2).unwrap().as_str().parse()?,
captures.get(3).unwrap().as_str().parse()?,
captures
.get(2)
.unwrap()
.as_str()
.parse()
.or(Err(DownloadError::Progressive(
"could not parse range header number".into(),
)))?,
captures
.get(3)
.unwrap()
.as_str()
.parse()
.or(Err(DownloadError::Progressive(
"could not parse range header number".into(),
)))?,
))
}
@ -76,7 +95,8 @@ async fn download_single_file<P: Into<PathBuf>>(
let mut size: Option<u64> = None;
// If the url is from googlevideo, extract file size from clen parameter
let (url_base, url_params) = util::url_to_params(url)?;
let (url_base, url_params) =
util::url_to_params(url).or_else(|e| Err(DownloadError::Other(e.to_string().into())))?;
let is_gvideo = url_base.ends_with(".googlevideo.com/videoplayback");
if is_gvideo {
size = url_params.get("clen").and_then(|s| s.parse::<u64>().ok());
@ -95,9 +115,14 @@ async fn download_single_file<P: Into<PathBuf>>(
let cr_header = some_or_bail!(
res.headers().get(header::CONTENT_RANGE),
Err(anyhow!("Did not get Content-Range header"))
Err(DownloadError::Progressive(
"Did not get Content-Range header".into()
))
)
.to_str()?;
.to_str()
.or(Err(DownloadError::Progressive(
"could not convert Content-Range header to string".into(),
)))?;
let (_, original_size) = parse_cr_header(cr_header)?;
@ -117,9 +142,12 @@ async fn download_single_file<P: Into<PathBuf>>(
}
Ordering::Greater => {
// WTF?
return Err(anyhow!(
"Already downloaded file {} is larger than original",
output_path_tmp.to_str().unwrap_or_default()
return Err(DownloadError::Other(
format!(
"Already downloaded file {} is larger than original",
output_path_tmp.to_str().unwrap_or_default()
)
.into(),
));
}
}
@ -174,9 +202,14 @@ async fn download_chunks_by_header(
// Content-Range: bytes 0-100/451368980
let cr_header = some_or_bail!(
res.headers().get(header::CONTENT_RANGE),
Err(anyhow!("Did not get Content-Range header"))
Err(DownloadError::Progressive(
"Did not get Content-Range header".into()
))
)
.to_str()?;
.to_str()
.or(Err(DownloadError::Progressive(
"could not convert Content-Range header to string".into(),
)))?;
let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?;
@ -279,7 +312,7 @@ pub async fn download_video(
let (video, audio) = player_data.select_video_audio_stream(filter);
if video.is_none() && audio.is_none() {
return Err(anyhow!("no stream found"));
return Err(DownloadError::Input("no stream found".into()));
}
let format = output_format.unwrap_or(
@ -287,7 +320,9 @@ pub async fn download_video(
Some(_) => "mp4",
None => match audio {
Some(audio) => match audio.codec {
AudioCodec::Unknown => return Err(anyhow!("unknown audio codec")),
AudioCodec::Unknown => {
return Err(DownloadError::Input("unknown audio codec".into()))
}
AudioCodec::Mp4a => "m4a",
AudioCodec::Opus => "opus",
},
@ -302,7 +337,9 @@ pub async fn download_video(
// If the downloaded video already exists, only error if the download path was
// chosen explicitly.
if output_fname_set {
bail!("File {} already exists", output_path.to_string_lossy());
return Err(DownloadError::Input(
format!("File {} already exists", output_path.to_string_lossy()).into(),
))?;
} else {
info!(
"Downloaded video {} already exists",
@ -366,7 +403,7 @@ pub async fn download_video(
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<_, _>>()?;
.collect::<core::result::Result<_, _>>()?;
}
}
@ -423,10 +460,13 @@ async fn convert_streams<P: Into<PathBuf>>(
let res = Command::new(ffmpeg).args(args).output().await?;
if !res.status.success() {
bail!(
"ffmpeg error: {}",
std::str::from_utf8(&res.stderr).unwrap_or_default()
)
return Err(DownloadError::Ffmpeg(
format!(
"ffmpeg error: {}",
std::str::from_utf8(&res.stderr).unwrap_or_default()
)
.into(),
));
}
Ok(())
}

91
src/error.rs Normal file
View file

@ -0,0 +1,91 @@
use std::borrow::Cow;
pub(crate) type Result<T> = core::result::Result<T, Error>;
/// Custom error type for the RustyPipe library
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum Error {
/// Error extracting content from YouTube
#[error("extraction error: {0}")]
Extraction(#[from] ExtractionError),
/// Error from the deobfuscater
#[error("deobfuscator error: {0}")]
Deobfuscation(#[from] DeobfError),
/// Error from the video downloader
#[error("download error: {0}")]
Download(#[from] DownloadError),
/// File IO error
#[error(transparent)]
Io(#[from] std::io::Error),
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
#[error("http status code: {0}")]
HttpStatus(u16),
#[error("error: {0}")]
Other(Cow<'static, str>),
}
/// Error that occurred during the initialization
/// or use of the YouTube URL signature deobfuscator.
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum DeobfError {
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
/// Error during JavaScript execution
#[error("js execution error: {0}")]
JavaScript(#[from] quick_js::ExecutionError),
#[error("js parsing: {0}")]
JsParser(#[from] ress::error::Error),
/// Could not extract certain data
#[error("could not extract {0}")]
Extraction(&'static str),
#[error("error: {0}")]
Other(&'static str),
}
/// Error from the video downloader
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum DownloadError {
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
/// File IO error
#[error(transparent)]
Io(#[from] std::io::Error),
#[error("FFmpeg error: {0}")]
Ffmpeg(Cow<'static, str>),
#[error("Progressive download error: {0}")]
Progressive(Cow<'static, str>),
#[error("input error: {0}")]
Input(Cow<'static, str>),
#[error("error: {0}")]
Other(Cow<'static, str>),
}
/// Error extracting content from YouTube
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum ExtractionError {
#[error("Video cant be played because of {0}. Reason (from YT): {1}")]
VideoUnavailable(&'static str, String),
#[error("Video is age restricted")]
VideoAgeRestricted,
#[error("deserialization error: {0}")]
Deserialization(#[from] serde_json::Error),
#[error("got invalid data from YT: {0}")]
InvalidData(Cow<'static, str>),
#[error("got wrong result from YT: {0}")]
WrongResult(String),
#[error("Warnings during deserialization/mapping")]
Warnings,
}
/// Internal error
#[derive(thiserror::Error, Debug)]
#[error("mapping error: {0}")]
pub struct MappingError(pub(crate) Cow<'static, str>);

1
src/lib.rs
View file

@ -17,6 +17,7 @@ mod util;
pub mod cache;
pub mod client;
pub mod download;
pub mod error;
pub mod model;
pub mod report;
pub mod timeago;

10
src/report.rs
View file

@ -6,12 +6,12 @@ use std::{
path::{Path, PathBuf},
};
use anyhow::Result;
use chrono::{DateTime, Local};
use log::error;
use serde::{Deserialize, Serialize};
use crate::deobfuscate::DeobfData;
use crate::error::{Error, Result};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
@ -81,7 +81,7 @@ impl Default for Info {
}
}
pub trait Reporter {
pub trait Reporter: Sync + Send {
fn report(&self, report: &Report);
}
@ -98,7 +98,11 @@ impl FileReporter {
fn _report(&self, report: &Report) -> Result<()> {
let report_path = get_report_path(&self.path, report, "json")?;
serde_json::to_writer_pretty(&File::create(report_path)?, &report)?;
serde_json::to_writer_pretty(&File::create(report_path)?, &report).or_else(|e| {
Err(Error::Other(
format!("could not serialize report. err: {}", e).into(),
))
})?;
Ok(())
}
}

9
src/serializer/text.rs
View file

@ -1,12 +1,11 @@
use std::convert::TryFrom;
use anyhow::anyhow;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DefaultOnError, DeserializeAs};
use crate::util;
use crate::{error::MappingError, util};
/// # Text
///
@ -366,7 +365,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText {
}
impl TryFrom<TextComponent> for crate::model::ChannelId {
type Error = anyhow::Error;
type Error = MappingError;
fn try_from(value: TextComponent) -> Result<Self, Self::Error> {
match value {
@ -379,9 +378,9 @@ impl TryFrom<TextComponent> for crate::model::ChannelId {
id: browse_id,
name: text,
}),
_ => Err(anyhow!("invalid channel link type")),
_ => Err(MappingError("invalid channel link type".into())),
},
_ => Err(anyhow!("invalid channel link")),
_ => Err(MappingError("invalid channel link".into())),
}
}
}

11
src/util.rs
View file

@ -1,12 +1,11 @@
use std::{borrow::Borrow, collections::BTreeMap, str::FromStr};
use anyhow::Result;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use rand::Rng;
use url::Url;
use crate::{dictionary, model::Language};
use crate::{dictionary, error::Error, error::Result, model::Language};
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@ -44,7 +43,11 @@ pub fn generate_content_playback_nonce() -> String {
///
/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}`
pub fn url_to_params(url: &str) -> Result<(String, BTreeMap<String, String>)> {
let mut parsed_url = Url::parse(url)?;
let mut parsed_url = Url::parse(url).or_else(|e| {
Err(Error::Other(
format!("could not parse url `{}` err: {}", url, e).into(),
))
})?;
let url_params: BTreeMap<String, String> = parsed_url
.query_pairs()
.map(|(k, v)| (k.to_string(), v.to_string()))
@ -56,7 +59,7 @@ pub fn url_to_params(url: &str) -> Result<(String, BTreeMap<String, String>)> {
}
/// Parse a string after removing all non-numeric characters
pub fn parse_numeric<F>(string: &str) -> Result<F, F::Err>
pub fn parse_numeric<F>(string: &str) -> core::result::Result<F, F::Err>
where
F: FromStr,
{