From a3e3269fb322712bc746b88146553d3d86d930ce Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sat, 8 Oct 2022 14:30:09 +0200 Subject: [PATCH] feat: add custom error types, remove anyhow --- Cargo.toml | 8 ++-- src/cache.rs | 2 +- src/client/channel.rs | 37 ++++++++------- src/client/channel_rss.rs | 15 ++++-- src/client/mod.rs | 73 +++++++++++++++-------------- src/client/pagination.rs | 2 +- src/client/player.rs | 58 ++++++++++++++--------- src/client/playlist.rs | 40 ++++++++++------ src/client/video_details.rs | 46 +++++++++++++------ src/deobfuscate.rs | 82 +++++++++++++++++---------------- src/download.rs | 84 +++++++++++++++++++++++++--------- src/error.rs | 91 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/report.rs | 10 ++-- src/serializer/text.rs | 9 ++-- src/util.rs | 11 +++-- 16 files changed, 385 insertions(+), 184 deletions(-) create mode 100644 src/error.rs diff --git a/Cargo.toml b/Cargo.toml index f1222bc..b79de1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ include = ["/src", "README.md", "LICENSE", "!snapshots"] members = [".", "codegen", "cli"] [features] -default = ["default-tls"] +default = ["default-tls", "rss"] all = ["rss", "html"] rss = ["quick-xml"] @@ -26,10 +26,10 @@ rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"] [dependencies] # quick-js = "0.4.1" -quick-js = { path = "../quickjs-rs" } +quick-js = { path = "../quickjs-rs", default-features = false } once_cell = "1.12.0" fancy-regex = "0.10.0" -anyhow = "1.0" +thiserror = "1.0.36" url = "2.2.2" log = "0.4.17" reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]} @@ -38,7 +38,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" serde_with = {version = "2.0.0", features = ["json"] } rand = "0.8.5" -chrono = {version = "0.4.19", features = ["serde"]} +chrono = {version = "0.4.19", default-features = false, features = ["clock", "serde"]} chronoutil = "0.2.3" futures = "0.3.21" indicatif = "0.17.0" diff --git a/src/cache.rs b/src/cache.rs index 633599e..fea15a8 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -7,7 +7,7 @@ use std::{ use log::error; -pub trait CacheStorage { +pub trait CacheStorage: Sync + Send { fn write(&self, data: &str); fn read(&self) -> Option; } diff --git a/src/client/channel.rs b/src/client/channel.rs index 16968d8..4ea2c4f 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -1,9 +1,9 @@ -use anyhow::{anyhow, bail, Result}; use chrono::TimeZone; use serde::Serialize; use url::Url; use crate::{ + error::{Error, ExtractionError}, model::{ Channel, ChannelInfo, ChannelOrder, ChannelPlaylist, ChannelVideo, Language, Paginator, }, @@ -43,7 +43,7 @@ impl RustyPipeQuery { pub async fn channel_videos( &self, channel_id: &str, - ) -> Result>> { + ) -> Result>, Error> { self.channel_videos_ordered(channel_id, ChannelOrder::default()) .await } @@ -52,7 +52,7 @@ impl RustyPipeQuery { &self, channel_id: &str, order: ChannelOrder, - ) -> Result>> { + ) -> Result>, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, @@ -77,7 +77,7 @@ impl RustyPipeQuery { pub async fn channel_videos_continuation( &self, ctoken: &str, - ) -> Result> { + ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, @@ -97,7 +97,7 @@ impl RustyPipeQuery { pub async fn channel_playlists( &self, channel_id: &str, - ) -> Result>> { + ) -> Result>, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, @@ -118,7 +118,7 @@ impl RustyPipeQuery { pub async fn channel_playlists_continuation( &self, ctoken: &str, - ) -> Result> { + ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, @@ -135,7 +135,7 @@ impl RustyPipeQuery { .await } - pub async fn channel_info(&self, channel_id: &str) -> Result> { + pub async fn channel_info(&self, channel_id: &str) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, @@ -160,7 +160,7 @@ impl MapResponse>> for response::Channel { id: &str, lang: crate::model::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>>> { + ) -> Result>>, ExtractionError> { let content = map_channel_content(self.contents, id); let mut warnings = content.warnings; let grid = match content.c { @@ -191,7 +191,7 @@ impl MapResponse>> for response::Channel { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>>> { + ) -> Result>>, ExtractionError> { let content = map_channel_content(self.contents, id); let mut warnings = content.warnings; let grid = match content.c { @@ -222,7 +222,7 @@ impl MapResponse> for response::Channel { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let content = map_channel_content(self.contents, id); let mut warnings = content.warnings; let meta = match content.c { @@ -278,11 +278,11 @@ impl MapResponse> for response::ChannelCont { _id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; let res = some_or_bail!( actions.try_swap_remove(0), - Err(anyhow!("no received action")) + Err(ExtractionError::InvalidData("no received action".into())) ) .append_continuation_items_action .continuation_items; @@ -297,11 +297,11 @@ impl MapResponse> for response::ChannelCont { _id: &str, _lang: Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; let res = some_or_bail!( actions.try_swap_remove(0), - Err(anyhow!("no received action")) + Err(ExtractionError::InvalidData("no received action".into())) ) .append_continuation_items_action .continuation_items; @@ -423,15 +423,14 @@ fn map_channel( content: T, id: &str, lang: Language, -) -> Result> { +) -> Result, ExtractionError> { let header = header.c4_tabbed_header_renderer; if header.channel_id != id { - bail!( + return Err(ExtractionError::WrongResult(format!( "got wrong channel id {}, expected {}", - header.channel_id, - id - ); + header.channel_id, id + ))); } Ok(Channel { diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index 3525530..33c6e6a 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -1,13 +1,15 @@ use std::collections::BTreeMap; -use anyhow::Result; - -use crate::{model::ChannelRss, report::Report}; +use crate::{ + error::{Error, ExtractionError}, + model::ChannelRss, + report::Report, +}; use super::{response, RustyPipeQuery}; impl RustyPipeQuery { - pub async fn channel_rss(&self, channel_id: &str) -> Result { + pub async fn channel_rss(&self, channel_id: &str) -> Result { let url = format!( "https://www.youtube.com/feeds/videos.xml?channel_id={}", channel_id @@ -41,7 +43,10 @@ impl RustyPipeQuery { reporter.report(&report); } - Err(e.into()) + Err(ExtractionError::InvalidData( + format!("could not deserialize xml: {}", e).into(), + ) + .into()) } } } diff --git a/src/client/mod.rs b/src/client/mod.rs index fec8c0a..4c5a549 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -14,7 +14,6 @@ mod channel_rss; use std::fmt::Debug; use std::sync::Arc; -use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Duration, Utc}; use fancy_regex::Regex; use log::{debug, error, warn}; @@ -27,6 +26,7 @@ use tokio::sync::RwLock; use crate::{ cache::{CacheStorage, FileStorage}, deobfuscate::{DeobfData, Deobfuscator}, + error::{Error, ExtractionError, Result}, model::{Country, Language}, report::{FileReporter, Level, Report, Reporter}, serializer::MapResult, @@ -166,8 +166,8 @@ pub struct RustyPipe { struct RustyPipeRef { http: Client, - storage: Option>, - reporter: Option>, + storage: Option>, + reporter: Option>, n_retries: u32, consent_cookie: String, cache: CacheHolder, @@ -183,8 +183,8 @@ struct RustyPipeOpts { } pub struct RustyPipeBuilder { - storage: Option>, - reporter: Option>, + storage: Option>, + reporter: Option>, n_retries: u32, user_agent: String, default_opts: RustyPipeOpts, @@ -452,8 +452,11 @@ impl RustyPipe { } /// Execute the given http request. - async fn http_request(&self, request: Request) -> Result { - let mut last_res: Option> = None; + async fn http_request( + &self, + request: Request, + ) -> core::result::Result { + let mut last_res = None; for n in 0..self.inner.n_retries { let res = self.inner.http.execute(request.try_clone().unwrap()).await; let emsg = match &res { @@ -509,11 +512,13 @@ impl RustyPipe { .build() .unwrap(), ) - .await - .context("Failed to download sw.js")?; + .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from( + ExtractionError::InvalidData( + "Could not find desktop client version in sw.js".into(), + ), + )) }; let from_html = async { @@ -525,11 +530,13 @@ impl RustyPipe { .build() .unwrap(), ) - .await - .context("Failed to get YT Desktop page")?; + .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from( + ExtractionError::InvalidData( + "Could not find desktop client version in sw.js".into(), + ), + )) }; match from_swjs.await { @@ -552,11 +559,11 @@ impl RustyPipe { .build() .unwrap(), ) - .await - .context("Failed to download sw.js")?; + .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version in sw.js")) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from( + ExtractionError::InvalidData("Could not find music client version in sw.js".into()), + )) }; let from_html = async { @@ -568,11 +575,13 @@ impl RustyPipe { .build() .unwrap(), ) - .await - .context("Failed to get YT Desktop page")?; + .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1) - .ok_or_else(|| anyhow!("Could not find desktop client version on html page")) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from( + ExtractionError::InvalidData( + "Could not find music client version on html page".into(), + ), + )) }; match from_swjs.await { @@ -971,7 +980,7 @@ impl RustyPipeQuery { }; if status.is_client_error() || status.is_server_error() { - let e = anyhow!("Server responded with error code {}", status); + let e = Error::HttpStatus(status.into()); create_report(Level::ERR, Some(e.to_string()), vec![]); return Err(e); } @@ -982,12 +991,12 @@ impl RustyPipeQuery { if !mapres.warnings.is_empty() { create_report( Level::WRN, - Some("Warnings during deserialization/mapping".to_owned()), + Some(ExtractionError::Warnings.to_string()), mapres.warnings, ); if self.opts.strict { - bail!("Warnings during deserialization/mapping"); + return Err(Error::Extraction(ExtractionError::Warnings)); } } else if self.opts.report { create_report(Level::DBG, None, vec![]); @@ -995,15 +1004,13 @@ impl RustyPipeQuery { Ok(mapres.c) } Err(e) => { - let emsg = "Could not map reponse"; - create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); - Err(e).context(emsg) + create_report(Level::ERR, Some(e.to_string()), Vec::new()); + Err(e.into()) } }, Err(e) => { - let emsg = "Could not deserialize response"; - create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]); - Err(e).context(emsg) + create_report(Level::ERR, Some(e.to_string()), Vec::new()); + Err(Error::from(ExtractionError::from(e))) } } } @@ -1058,7 +1065,7 @@ trait MapResponse { id: &str, lang: Language, deobf: Option<&Deobfuscator>, - ) -> Result>; + ) -> core::result::Result, crate::error::ExtractionError>; } #[cfg(test)] diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 30c9718..ae32195 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use crate::error::Result; use crate::model::{ ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, diff --git a/src/client/player.rs b/src/client/player.rs index 3f2ec41..457bdd2 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -3,7 +3,6 @@ use std::{ collections::{BTreeMap, HashMap}, }; -use anyhow::{anyhow, bail, Result}; use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone}; use fancy_regex::Regex; use once_cell::sync::Lazy; @@ -12,6 +11,7 @@ use url::Url; use crate::{ deobfuscate::Deobfuscator, + error::{DeobfError, Error, ExtractionError}, model::{ AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Language, Subtitle, VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream, @@ -58,7 +58,11 @@ struct QContentPlaybackContext { } impl RustyPipeQuery { - pub async fn player(self, video_id: &str, client_type: ClientType) -> Result { + pub async fn player( + self, + video_id: &str, + client_type: ClientType, + ) -> Result { let q1 = self.clone(); let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); let q2 = self.client.clone(); @@ -111,7 +115,7 @@ impl MapResponse for response::Player { id: &str, _lang: Language, deobf: Option<&Deobfuscator>, - ) -> Result> { + ) -> Result, ExtractionError> { let deobf = deobf.unwrap(); let mut warnings = vec![]; @@ -121,26 +125,36 @@ impl MapResponse for response::Player { live_streamability.is_some() } response::player::PlayabilityStatus::Unplayable { reason } => { - bail!("Video is unplayable. Reason: {}", reason) + return Err(ExtractionError::VideoUnavailable("DRM/Geoblock", reason)) } response::player::PlayabilityStatus::LoginRequired { reason } => { - bail!("Playback requires login. Reason: {}", reason) + // reason: "Sign in to confirm your age" + if reason.split_whitespace().any(|word| word == "age") { + return Err(ExtractionError::VideoAgeRestricted); + } + return Err(ExtractionError::VideoUnavailable("private video", reason)); } response::player::PlayabilityStatus::LiveStreamOffline { reason } => { - bail!("Livestream is offline. Reason: {}", reason) + return Err(ExtractionError::VideoUnavailable( + "offline livestream", + reason, + )) } response::player::PlayabilityStatus::Error { reason } => { - bail!("Video was deleted. Reason: {}", reason) + return Err(ExtractionError::VideoUnavailable( + "deletion/censorship", + reason, + )) } }; let mut streaming_data = some_or_bail!( self.streaming_data, - Err(anyhow!("No streaming data was returned")) + Err(ExtractionError::InvalidData("no streaming data".into())) ); let video_details = some_or_bail!( self.video_details, - Err(anyhow!("No video details were returned")) + Err(ExtractionError::InvalidData("no video details".into())) ); let microformat = self.microformat.map(|m| m.player_microformat_renderer); let (publish_date, category, tags, is_family_safe) = @@ -159,11 +173,10 @@ impl MapResponse for response::Player { }); if video_details.video_id != id { - bail!( - "got wrong video id {}, expected {}", - video_details.video_id, - id - ); + return Err(ExtractionError::WrongResult(format!( + "video id {}, expected {}", + video_details.video_id, id + ))); } let video_info = VideoPlayerDetails { @@ -272,7 +285,7 @@ impl MapResponse for response::Player { fn cipher_to_url_params( signature_cipher: &str, deobf: &Deobfuscator, -) -> Result<(String, BTreeMap)> { +) -> Result<(String, BTreeMap), DeobfError> { let params: HashMap, Cow> = url::form_urlencoded::parse(signature_cipher.as_bytes()).collect(); @@ -281,12 +294,15 @@ fn cipher_to_url_params( // `sp`: Signature parameter // `url`: URL that is missing the signature parameter - let sig = some_or_bail!(params.get("s"), Err(anyhow!("no s param"))); - let sp = some_or_bail!(params.get("sp"), Err(anyhow!("no sp param"))); - let raw_url = some_or_bail!(params.get("url"), Err(anyhow!("no url param"))); - let (url_base, mut url_params) = util::url_to_params(raw_url)?; + let sig = some_or_bail!(params.get("s"), Err(DeobfError::Extraction("s param"))); + let sp = some_or_bail!(params.get("sp"), Err(DeobfError::Extraction("sp param"))); + let raw_url = some_or_bail!( + params.get("url"), + Err(DeobfError::Extraction("no url param")) + ); + let (url_base, mut url_params) = + util::url_to_params(raw_url).or(Err(DeobfError::Extraction("url params")))?; - // println!("sig: {}", sig); let deobf_sig = deobf.deobfuscate_sig(sig)?; url_params.insert(sp.to_string(), deobf_sig); @@ -297,7 +313,7 @@ fn deobf_nsig( url_params: &mut BTreeMap, deobf: &Deobfuscator, last_nsig: &mut [String; 2], -) -> Result<()> { +) -> Result<(), DeobfError> { let nsig: String; if let Some(n) = url_params.get("n") { nsig = if n == &last_nsig[0] { diff --git a/src/client/playlist.rs b/src/client/playlist.rs index bd85aa8..c9e6095 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -1,10 +1,10 @@ use std::convert::TryFrom; -use anyhow::{anyhow, bail, Result}; use serde::Serialize; use crate::{ deobfuscate::Deobfuscator, + error::{Error, ExtractionError}, model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo}, timeago, util::{self, TryRemove}, @@ -22,7 +22,7 @@ struct QPlaylist { } impl RustyPipeQuery { - pub async fn playlist(self, playlist_id: &str) -> Result { + pub async fn playlist(self, playlist_id: &str) -> Result { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QPlaylist { context, @@ -39,7 +39,10 @@ impl RustyPipeQuery { .await } - pub async fn playlist_continuation(self, ctoken: &str) -> Result> { + pub async fn playlist_continuation( + self, + ctoken: &str, + ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, @@ -63,26 +66,32 @@ impl MapResponse for response::Playlist { id: &str, lang: Language, _deobf: Option<&Deobfuscator>, - ) -> Result> { + ) -> Result, ExtractionError> { // TODO: think about a deserializer that deserializes only first list item let mut tcbr_contents = self.contents.two_column_browse_results_renderer.contents; let video_items = some_or_bail!( some_or_bail!( some_or_bail!( tcbr_contents.try_swap_remove(0), - Err(anyhow!("twoColumnBrowseResultsRenderer empty")) + Err(ExtractionError::InvalidData( + "twoColumnBrowseResultsRenderer empty".into() + )) ) .tab_renderer .content .section_list_renderer .contents .try_swap_remove(0), - Err(anyhow!("sectionListRenderer empty")) + Err(ExtractionError::InvalidData( + "sectionListRenderer empty".into() + )) ) .item_section_renderer .contents .try_swap_remove(0), - Err(anyhow!("itemSectionRenderer empty")) + Err(ExtractionError::InvalidData( + "itemSectionRenderer empty".into() + )) ) .playlist_video_list_renderer .contents; @@ -94,7 +103,7 @@ impl MapResponse for response::Playlist { let mut sidebar_items = sidebar.playlist_sidebar_renderer.items; let mut primary = some_or_bail!( sidebar_items.try_swap_remove(0), - Err(anyhow!("no primary sidebar")) + Err(ExtractionError::InvalidData("no primary sidebar".into())) ); ( @@ -112,7 +121,7 @@ impl MapResponse for response::Playlist { None => { let header_banner = some_or_bail!( self.header.playlist_header_renderer.playlist_header_banner, - Err(anyhow!("no thumbnail found")) + Err(ExtractionError::InvalidData("no thumbnail found".into())) ); let mut byline = self.header.playlist_header_renderer.byline; @@ -131,7 +140,7 @@ impl MapResponse for response::Playlist { Some(_) => { ok_or_bail!( util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text), - Err(anyhow!("no video count")) + Err(ExtractionError::InvalidData("no video count".into())) ) } None => videos.len() as u32, @@ -139,7 +148,10 @@ impl MapResponse for response::Playlist { let playlist_id = self.header.playlist_header_renderer.playlist_id; if playlist_id != id { - bail!("got wrong playlist id {}, expected {}", playlist_id, id); + return Err(ExtractionError::WrongResult(format!( + "got wrong playlist id {}, expected {}", + playlist_id, id + ))); } let name = self.header.playlist_header_renderer.title; @@ -178,11 +190,13 @@ impl MapResponse> for response::PlaylistCont { _id: &str, _lang: Language, _deobf: Option<&Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let mut actions = self.on_response_received_actions; let action = some_or_bail!( actions.try_swap_remove(0), - Err(anyhow!("no continuation action")) + Err(ExtractionError::InvalidData( + "no continuation action".into() + )) ); let (items, ctoken) = diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 62a1bea..808f759 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -1,9 +1,9 @@ use std::convert::TryFrom; -use anyhow::{anyhow, bail, Result}; use serde::Serialize; use crate::{ + error::{Error, ExtractionError}, model::{ ChannelId, ChannelTag, Chapter, Comment, Language, Paginator, RecommendedVideo, VideoDetails, @@ -30,7 +30,7 @@ struct QVideo { } impl RustyPipeQuery { - pub async fn video_details(self, video_id: &str) -> Result { + pub async fn video_details(self, video_id: &str) -> Result { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QVideo { context, @@ -49,7 +49,10 @@ impl RustyPipeQuery { .await } - pub async fn video_recommendations(self, ctoken: &str) -> Result> { + pub async fn video_recommendations( + self, + ctoken: &str, + ) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, @@ -66,7 +69,7 @@ impl RustyPipeQuery { .await } - pub async fn video_comments(self, ctoken: &str) -> Result> { + pub async fn video_comments(self, ctoken: &str) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, @@ -90,12 +93,15 @@ impl MapResponse for response::VideoDetails { id: &str, lang: crate::model::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result> { + ) -> Result, ExtractionError> { let mut warnings = Vec::new(); let video_id = self.current_video_endpoint.watch_endpoint.video_id; if id != video_id { - bail!("got wrong playlist id {}, expected {}", video_id, id); + return Err(ExtractionError::WrongResult(format!( + "got wrong playlist id {}, expected {}", + video_id, id + ))); } let mut primary_results = self @@ -167,7 +173,11 @@ impl MapResponse for response::VideoDetails { view_count.video_view_count_renderer.is_live, ) } - _ => bail!("could not find primary_info"), + _ => { + return Err(ExtractionError::InvalidData( + "could not find primary_info".into(), + )) + } }; let comment_count = comment_count_section.and_then(|s| { @@ -214,7 +224,11 @@ impl MapResponse for response::VideoDetails { (owner.video_owner_renderer, desc, is_ccommons) } - _ => bail!("could not find secondary_info"), + _ => { + return Err(ExtractionError::InvalidData( + "could not find secondary_info".into(), + )) + } }; let (channel_id, channel_name) = match owner.title { @@ -224,9 +238,13 @@ impl MapResponse for response::VideoDetails { browse_id, } => match page_type { crate::serializer::text::PageType::Channel => (browse_id, text), - _ => bail!("invalid channel link type"), + _ => { + return Err(ExtractionError::InvalidData( + "invalid channel link type".into(), + )) + } }, - _ => bail!("invalid channel link"), + _ => return Err(ExtractionError::InvalidData("invalid channel link".into())), }; let recommended = self @@ -324,11 +342,13 @@ impl MapResponse> for response::VideoRecommendations _id: &str, lang: crate::model::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let mut endpoints = self.on_response_received_endpoints; let cont = some_or_bail!( endpoints.try_swap_remove(0), - Err(anyhow!("no continuation endpoint")) + Err(ExtractionError::InvalidData( + "no continuation endpoint".into() + )) ); Ok(map_recommendations( @@ -344,7 +364,7 @@ impl MapResponse> for response::VideoComments { _id: &str, lang: crate::model::Language, _deobf: Option<&crate::deobfuscate::Deobfuscator>, - ) -> Result>> { + ) -> Result>, ExtractionError> { let mut warnings = self.on_response_received_endpoints.warnings; let mut comments = Vec::new(); diff --git a/src/deobfuscate.rs b/src/deobfuscate.rs index 23e12f0..961419e 100644 --- a/src/deobfuscate.rs +++ b/src/deobfuscate.rs @@ -1,4 +1,3 @@ -use anyhow::{anyhow, bail, Context, Result}; use fancy_regex::Regex; use log::debug; use once_cell::sync::Lazy; @@ -6,7 +5,9 @@ use reqwest::Client; use serde::{Deserialize, Serialize}; use std::result::Result::Ok; -use crate::util; +use crate::{error::DeobfError, util}; + +type Result = core::result::Result; pub struct Deobfuscator { data: DeobfData, @@ -22,13 +23,8 @@ pub struct DeobfData { impl Deobfuscator { pub async fn new(http: Client) -> Result { - let js_url = get_player_js_url(&http) - .await - .context("Failed to retrieve player.js URL")?; - - let player_js = get_response(&http, &js_url) - .await - .context("Failed to download player.js")?; + let js_url = get_player_js_url(&http).await?; + let player_js = get_response(&http, &js_url).await?; debug!("Downloaded player.js from {}", js_url); @@ -84,7 +80,7 @@ fn get_sig_fn_name(player_js: &str) -> Result { }); util::get_cg_from_regexes(FUNCTION_PATTERNS.iter(), player_js, 1) - .ok_or_else(|| anyhow!("could not find deobf function name")) + .ok_or(DeobfError::Extraction("deobf function name")) } fn caller_function(fn_name: &str) -> String { @@ -98,13 +94,13 @@ fn get_sig_fn(player_js: &str) -> Result { "(".to_owned() + &dfunc_name.replace('$', "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})"; let function_pattern = ok_or_bail!( Regex::new(&function_pattern_str), - Err(anyhow!("could not parse function pattern regex")) + Err(DeobfError::Other("could not parse function pattern regex")) ); let deobfuscate_function = "var ".to_owned() + some_or_bail!( function_pattern.captures(player_js).ok().flatten(), - Err(anyhow!("could not find deobf function")) + Err(DeobfError::Extraction("deobf function")) ) .get(1) .unwrap() @@ -118,7 +114,7 @@ fn get_sig_fn(player_js: &str) -> Result { .captures(&deobfuscate_function) .ok() .flatten(), - Err(anyhow!("could not find helper object name")) + Err(DeobfError::Extraction("helper object name")) ) .get(1) .unwrap() @@ -128,12 +124,12 @@ fn get_sig_fn(player_js: &str) -> Result { "(var ".to_owned() + &helper_object_name.replace('$', "\\$") + "=\\{.+?\\}\\};)"; let helper_pattern = ok_or_bail!( Regex::new(&helper_pattern_str), - Err(anyhow!("could not parse helper pattern regex")) + Err(DeobfError::Other("could not parse helper pattern regex")) ); let player_js_nonl = player_js.replace('\n', ""); let helper_object = some_or_bail!( helper_pattern.captures(&player_js_nonl).ok().flatten(), - Err(anyhow!("could not find helper object")) + Err(DeobfError::Extraction("helper object")) ) .get(1) .unwrap() @@ -143,14 +139,15 @@ fn get_sig_fn(player_js: &str) -> Result { } fn deobfuscate_sig(sig: &str, sig_fn: &str) -> Result { - let context = quick_js::Context::new()?; + let context = + quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?; context.eval(sig_fn)?; let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?; - match res.as_str() { - Some(res) => Ok(res.to_owned()), - None => bail!("deobfuscation func returned null"), - } + res.as_str().map_or( + Err(DeobfError::Other("sig deobfuscation func returned null")), + |res| Ok(res.to_owned()), + ) } fn get_nsig_fn_name(player_js: &str) -> Result { @@ -161,7 +158,7 @@ fn get_nsig_fn_name(player_js: &str) -> Result { let fname_match = some_or_bail!( FUNCTION_NAME_PATTERN.captures(player_js).ok().flatten(), - Err(anyhow!("could not find n_deobf function")) + Err(DeobfError::Extraction("n_deobf function")) ); let function_name = fname_match.get(1).unwrap().as_str(); @@ -170,25 +167,29 @@ fn get_nsig_fn_name(player_js: &str) -> Result { return Ok(function_name.to_owned()); } - let array_num = fname_match.get(2).unwrap().as_str().parse::()?; + let array_num = fname_match + .get(2) + .unwrap() + .as_str() + .parse::() + .or(Err(DeobfError::Other("could not parse array_num")))?; let array_pattern_str = "var ".to_owned() + &fancy_regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];"; - let array_pattern = Regex::new(&array_pattern_str)?; + let array_pattern = Regex::new(&array_pattern_str).or(Err(DeobfError::Other( + "could not parse helper pattern regex", + )))?; + let array_str = some_or_bail!( array_pattern.captures(player_js).ok().flatten(), - Err(anyhow!("could not find n_deobf array_str")) + Err(DeobfError::Extraction("n_deobf array_str")) ) .get(1) .unwrap() .as_str(); let mut names = array_str.split(','); let name = some_or_bail!( - names.nth(array_num.try_into()?), - Err(anyhow!( - "could not get {}th item from {}", - array_num, - array_str - )) + names.nth(array_num), + Err(DeobfError::Extraction("n_deobf function name")) ); Ok(name.to_owned()) } @@ -239,7 +240,7 @@ fn extract_js_fn(js: &str, name: &str) -> Result { } if state != 3 { - return Err(anyhow!("could not extract js fn")); + return Err(DeobfError::Extraction("javascript function")); } Ok(js[start..end].to_owned()) @@ -255,14 +256,15 @@ fn get_nsig_fn(player_js: &str) -> Result { } fn deobfuscate_nsig(sig: &str, nsig_fn: &str) -> Result { - let context = quick_js::Context::new()?; + let context = + quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?; context.eval(nsig_fn)?; let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?; - match res.as_str() { - Some(res) => Ok(res.to_owned()), - None => bail!("deobfuscation func returned null"), - } + res.as_str().map_or( + Err(DeobfError::Other("nsig deobfuscation func returned null")), + |res| Ok(res.to_owned()), + ) } async fn get_player_js_url(http: &Client) -> Result { @@ -278,8 +280,8 @@ async fn get_player_js_url(http: &Client) -> Result { .unwrap() }); let player_hash = some_or_bail!( - PLAYER_HASH_PATTERN.captures(&text)?, - Err(anyhow!("could not find player hash")) + PLAYER_HASH_PATTERN.captures(&text).ok().flatten(), + Err(DeobfError::Extraction("player hash")) ) .get(1) .unwrap() @@ -301,8 +303,8 @@ fn get_sts(player_js: &str) -> Result { Lazy::new(|| Regex::new("signatureTimestamp[=:](\\d+)").unwrap()); Ok(some_or_bail!( - STS_PATTERN.captures(player_js)?, - Err(anyhow!("could not find sts")) + STS_PATTERN.captures(player_js).ok().flatten(), + Err(DeobfError::Extraction("sts")) ) .get(1) .unwrap() diff --git a/src/download.rs b/src/download.rs index 3fed542..a9ce92f 100644 --- a/src/download.rs +++ b/src/download.rs @@ -2,7 +2,6 @@ use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf}; -use anyhow::{anyhow, bail, Result}; use fancy_regex::Regex; use futures::stream::{self, StreamExt}; use indicatif::ProgressBar; @@ -17,10 +16,13 @@ use tokio::{ }; use crate::{ + error::DownloadError, model::{stream_filter::Filter, AudioCodec, FileFormat, VideoCodec, VideoPlayer}, util, }; +type Result = core::result::Result; + const CHUNK_SIZE_MIN: u64 = 9000000; const CHUNK_SIZE_MAX: u64 = 10000000; @@ -44,15 +46,32 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { let captures = some_or_bail!( PATTERN.captures(cr_header).ok().flatten(), - Err(anyhow!( - "Content-Range header '{}' does not match pattern.", - cr_header + Err(DownloadError::Progressive( + format!( + "Content-Range header '{}' does not match pattern", + cr_header + ) + .into() )) ); Ok(( - captures.get(2).unwrap().as_str().parse()?, - captures.get(3).unwrap().as_str().parse()?, + captures + .get(2) + .unwrap() + .as_str() + .parse() + .or(Err(DownloadError::Progressive( + "could not parse range header number".into(), + )))?, + captures + .get(3) + .unwrap() + .as_str() + .parse() + .or(Err(DownloadError::Progressive( + "could not parse range header number".into(), + )))?, )) } @@ -76,7 +95,8 @@ async fn download_single_file>( let mut size: Option = None; // If the url is from googlevideo, extract file size from clen parameter - let (url_base, url_params) = util::url_to_params(url)?; + let (url_base, url_params) = + util::url_to_params(url).or_else(|e| Err(DownloadError::Other(e.to_string().into())))?; let is_gvideo = url_base.ends_with(".googlevideo.com/videoplayback"); if is_gvideo { size = url_params.get("clen").and_then(|s| s.parse::().ok()); @@ -95,9 +115,14 @@ async fn download_single_file>( let cr_header = some_or_bail!( res.headers().get(header::CONTENT_RANGE), - Err(anyhow!("Did not get Content-Range header")) + Err(DownloadError::Progressive( + "Did not get Content-Range header".into() + )) ) - .to_str()?; + .to_str() + .or(Err(DownloadError::Progressive( + "could not convert Content-Range header to string".into(), + )))?; let (_, original_size) = parse_cr_header(cr_header)?; @@ -117,9 +142,12 @@ async fn download_single_file>( } Ordering::Greater => { // WTF? - return Err(anyhow!( - "Already downloaded file {} is larger than original", - output_path_tmp.to_str().unwrap_or_default() + return Err(DownloadError::Other( + format!( + "Already downloaded file {} is larger than original", + output_path_tmp.to_str().unwrap_or_default() + ) + .into(), )); } } @@ -174,9 +202,14 @@ async fn download_chunks_by_header( // Content-Range: bytes 0-100/451368980 let cr_header = some_or_bail!( res.headers().get(header::CONTENT_RANGE), - Err(anyhow!("Did not get Content-Range header")) + Err(DownloadError::Progressive( + "Did not get Content-Range header".into() + )) ) - .to_str()?; + .to_str() + .or(Err(DownloadError::Progressive( + "could not convert Content-Range header to string".into(), + )))?; let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?; @@ -279,7 +312,7 @@ pub async fn download_video( let (video, audio) = player_data.select_video_audio_stream(filter); if video.is_none() && audio.is_none() { - return Err(anyhow!("no stream found")); + return Err(DownloadError::Input("no stream found".into())); } let format = output_format.unwrap_or( @@ -287,7 +320,9 @@ pub async fn download_video( Some(_) => "mp4", None => match audio { Some(audio) => match audio.codec { - AudioCodec::Unknown => return Err(anyhow!("unknown audio codec")), + AudioCodec::Unknown => { + return Err(DownloadError::Input("unknown audio codec".into())) + } AudioCodec::Mp4a => "m4a", AudioCodec::Opus => "opus", }, @@ -302,7 +337,9 @@ pub async fn download_video( // If the downloaded video already exists, only error if the download path was // chosen explicitly. if output_fname_set { - bail!("File {} already exists", output_path.to_string_lossy()); + return Err(DownloadError::Input( + format!("File {} already exists", output_path.to_string_lossy()).into(), + ))?; } else { info!( "Downloaded video {} already exists", @@ -366,7 +403,7 @@ pub async fn download_video( .collect::>() .await .into_iter() - .collect::>()?; + .collect::>()?; } } @@ -423,10 +460,13 @@ async fn convert_streams>( let res = Command::new(ffmpeg).args(args).output().await?; if !res.status.success() { - bail!( - "ffmpeg error: {}", - std::str::from_utf8(&res.stderr).unwrap_or_default() - ) + return Err(DownloadError::Ffmpeg( + format!( + "ffmpeg error: {}", + std::str::from_utf8(&res.stderr).unwrap_or_default() + ) + .into(), + )); } Ok(()) } diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..97b061c --- /dev/null +++ b/src/error.rs @@ -0,0 +1,91 @@ +use std::borrow::Cow; + +pub(crate) type Result = core::result::Result; + +/// Custom error type for the RustyPipe library +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum Error { + /// Error extracting content from YouTube + #[error("extraction error: {0}")] + Extraction(#[from] ExtractionError), + /// Error from the deobfuscater + #[error("deobfuscator error: {0}")] + Deobfuscation(#[from] DeobfError), + /// Error from the video downloader + #[error("download error: {0}")] + Download(#[from] DownloadError), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + #[error("http status code: {0}")] + HttpStatus(u16), + #[error("error: {0}")] + Other(Cow<'static, str>), +} + +/// Error that occurred during the initialization +/// or use of the YouTube URL signature deobfuscator. +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DeobfError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// Error during JavaScript execution + #[error("js execution error: {0}")] + JavaScript(#[from] quick_js::ExecutionError), + #[error("js parsing: {0}")] + JsParser(#[from] ress::error::Error), + /// Could not extract certain data + #[error("could not extract {0}")] + Extraction(&'static str), + #[error("error: {0}")] + Other(&'static str), +} + +/// Error from the video downloader +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DownloadError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("FFmpeg error: {0}")] + Ffmpeg(Cow<'static, str>), + #[error("Progressive download error: {0}")] + Progressive(Cow<'static, str>), + #[error("input error: {0}")] + Input(Cow<'static, str>), + #[error("error: {0}")] + Other(Cow<'static, str>), +} + +/// Error extracting content from YouTube +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum ExtractionError { + #[error("Video cant be played because of {0}. Reason (from YT): {1}")] + VideoUnavailable(&'static str, String), + #[error("Video is age restricted")] + VideoAgeRestricted, + #[error("deserialization error: {0}")] + Deserialization(#[from] serde_json::Error), + #[error("got invalid data from YT: {0}")] + InvalidData(Cow<'static, str>), + #[error("got wrong result from YT: {0}")] + WrongResult(String), + #[error("Warnings during deserialization/mapping")] + Warnings, +} + +/// Internal error +#[derive(thiserror::Error, Debug)] +#[error("mapping error: {0}")] +pub struct MappingError(pub(crate) Cow<'static, str>); diff --git a/src/lib.rs b/src/lib.rs index 114af58..da09f6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,7 @@ mod util; pub mod cache; pub mod client; pub mod download; +pub mod error; pub mod model; pub mod report; pub mod timeago; diff --git a/src/report.rs b/src/report.rs index 87b6f41..87e9d13 100644 --- a/src/report.rs +++ b/src/report.rs @@ -6,12 +6,12 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::Result; use chrono::{DateTime, Local}; use log::error; use serde::{Deserialize, Serialize}; use crate::deobfuscate::DeobfData; +use crate::error::{Error, Result}; #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] @@ -81,7 +81,7 @@ impl Default for Info { } } -pub trait Reporter { +pub trait Reporter: Sync + Send { fn report(&self, report: &Report); } @@ -98,7 +98,11 @@ impl FileReporter { fn _report(&self, report: &Report) -> Result<()> { let report_path = get_report_path(&self.path, report, "json")?; - serde_json::to_writer_pretty(&File::create(report_path)?, &report)?; + serde_json::to_writer_pretty(&File::create(report_path)?, &report).or_else(|e| { + Err(Error::Other( + format!("could not serialize report. err: {}", e).into(), + )) + })?; Ok(()) } } diff --git a/src/serializer/text.rs b/src/serializer/text.rs index b63d2b1..3406836 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -1,12 +1,11 @@ use std::convert::TryFrom; -use anyhow::anyhow; use fancy_regex::Regex; use once_cell::sync::Lazy; use serde::{Deserialize, Deserializer}; use serde_with::{serde_as, DefaultOnError, DeserializeAs}; -use crate::util; +use crate::{error::MappingError, util}; /// # Text /// @@ -366,7 +365,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText { } impl TryFrom for crate::model::ChannelId { - type Error = anyhow::Error; + type Error = MappingError; fn try_from(value: TextComponent) -> Result { match value { @@ -379,9 +378,9 @@ impl TryFrom for crate::model::ChannelId { id: browse_id, name: text, }), - _ => Err(anyhow!("invalid channel link type")), + _ => Err(MappingError("invalid channel link type".into())), }, - _ => Err(anyhow!("invalid channel link")), + _ => Err(MappingError("invalid channel link".into())), } } } diff --git a/src/util.rs b/src/util.rs index bd3e542..67980a2 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,12 +1,11 @@ use std::{borrow::Borrow, collections::BTreeMap, str::FromStr}; -use anyhow::Result; use fancy_regex::Regex; use once_cell::sync::Lazy; use rand::Rng; use url::Url; -use crate::{dictionary, model::Language}; +use crate::{dictionary, error::Error, error::Result, model::Language}; const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; @@ -44,7 +43,11 @@ pub fn generate_content_playback_nonce() -> String { /// /// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` pub fn url_to_params(url: &str) -> Result<(String, BTreeMap)> { - let mut parsed_url = Url::parse(url)?; + let mut parsed_url = Url::parse(url).or_else(|e| { + Err(Error::Other( + format!("could not parse url `{}` err: {}", url, e).into(), + )) + })?; let url_params: BTreeMap = parsed_url .query_pairs() .map(|(k, v)| (k.to_string(), v.to_string())) @@ -56,7 +59,7 @@ pub fn url_to_params(url: &str) -> Result<(String, BTreeMap)> { } /// Parse a string after removing all non-numeric characters -pub fn parse_numeric(string: &str) -> Result +pub fn parse_numeric(string: &str) -> core::result::Result where F: FromStr, {