feat: add custom error types, remove anyhow
This commit is contained in:
parent
1297bcb641
commit
a3e3269fb3
16 changed files with 385 additions and 184 deletions
|
|
@ -13,7 +13,7 @@ include = ["/src", "README.md", "LICENSE", "!snapshots"]
|
||||||
members = [".", "codegen", "cli"]
|
members = [".", "codegen", "cli"]
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["default-tls"]
|
default = ["default-tls", "rss"]
|
||||||
all = ["rss", "html"]
|
all = ["rss", "html"]
|
||||||
|
|
||||||
rss = ["quick-xml"]
|
rss = ["quick-xml"]
|
||||||
|
|
@ -26,10 +26,10 @@ rustls-tls-native-roots = ["reqwest/rustls-tls-native-roots"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# quick-js = "0.4.1"
|
# quick-js = "0.4.1"
|
||||||
quick-js = { path = "../quickjs-rs" }
|
quick-js = { path = "../quickjs-rs", default-features = false }
|
||||||
once_cell = "1.12.0"
|
once_cell = "1.12.0"
|
||||||
fancy-regex = "0.10.0"
|
fancy-regex = "0.10.0"
|
||||||
anyhow = "1.0"
|
thiserror = "1.0.36"
|
||||||
url = "2.2.2"
|
url = "2.2.2"
|
||||||
log = "0.4.17"
|
log = "0.4.17"
|
||||||
reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]}
|
reqwest = {version = "0.11.11", default-features = false, features = ["json", "gzip", "brotli", "stream"]}
|
||||||
|
|
@ -38,7 +38,7 @@ serde = { version = "1.0", features = ["derive"] }
|
||||||
serde_json = "1.0.82"
|
serde_json = "1.0.82"
|
||||||
serde_with = {version = "2.0.0", features = ["json"] }
|
serde_with = {version = "2.0.0", features = ["json"] }
|
||||||
rand = "0.8.5"
|
rand = "0.8.5"
|
||||||
chrono = {version = "0.4.19", features = ["serde"]}
|
chrono = {version = "0.4.19", default-features = false, features = ["clock", "serde"]}
|
||||||
chronoutil = "0.2.3"
|
chronoutil = "0.2.3"
|
||||||
futures = "0.3.21"
|
futures = "0.3.21"
|
||||||
indicatif = "0.17.0"
|
indicatif = "0.17.0"
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ use std::{
|
||||||
|
|
||||||
use log::error;
|
use log::error;
|
||||||
|
|
||||||
pub trait CacheStorage {
|
pub trait CacheStorage: Sync + Send {
|
||||||
fn write(&self, data: &str);
|
fn write(&self, data: &str);
|
||||||
fn read(&self) -> Option<String>;
|
fn read(&self) -> Option<String>;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
use anyhow::{anyhow, bail, Result};
|
|
||||||
use chrono::TimeZone;
|
use chrono::TimeZone;
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
error::{Error, ExtractionError},
|
||||||
model::{
|
model::{
|
||||||
Channel, ChannelInfo, ChannelOrder, ChannelPlaylist, ChannelVideo, Language, Paginator,
|
Channel, ChannelInfo, ChannelOrder, ChannelPlaylist, ChannelVideo, Language, Paginator,
|
||||||
},
|
},
|
||||||
|
|
@ -43,7 +43,7 @@ impl RustyPipeQuery {
|
||||||
pub async fn channel_videos(
|
pub async fn channel_videos(
|
||||||
&self,
|
&self,
|
||||||
channel_id: &str,
|
channel_id: &str,
|
||||||
) -> Result<Channel<Paginator<ChannelVideo>>> {
|
) -> Result<Channel<Paginator<ChannelVideo>>, Error> {
|
||||||
self.channel_videos_ordered(channel_id, ChannelOrder::default())
|
self.channel_videos_ordered(channel_id, ChannelOrder::default())
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
@ -52,7 +52,7 @@ impl RustyPipeQuery {
|
||||||
&self,
|
&self,
|
||||||
channel_id: &str,
|
channel_id: &str,
|
||||||
order: ChannelOrder,
|
order: ChannelOrder,
|
||||||
) -> Result<Channel<Paginator<ChannelVideo>>> {
|
) -> Result<Channel<Paginator<ChannelVideo>>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QChannel {
|
let request_body = QChannel {
|
||||||
context,
|
context,
|
||||||
|
|
@ -77,7 +77,7 @@ impl RustyPipeQuery {
|
||||||
pub async fn channel_videos_continuation(
|
pub async fn channel_videos_continuation(
|
||||||
&self,
|
&self,
|
||||||
ctoken: &str,
|
ctoken: &str,
|
||||||
) -> Result<Paginator<ChannelVideo>> {
|
) -> Result<Paginator<ChannelVideo>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QContinuation {
|
let request_body = QContinuation {
|
||||||
context,
|
context,
|
||||||
|
|
@ -97,7 +97,7 @@ impl RustyPipeQuery {
|
||||||
pub async fn channel_playlists(
|
pub async fn channel_playlists(
|
||||||
&self,
|
&self,
|
||||||
channel_id: &str,
|
channel_id: &str,
|
||||||
) -> Result<Channel<Paginator<ChannelPlaylist>>> {
|
) -> Result<Channel<Paginator<ChannelPlaylist>>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QChannel {
|
let request_body = QChannel {
|
||||||
context,
|
context,
|
||||||
|
|
@ -118,7 +118,7 @@ impl RustyPipeQuery {
|
||||||
pub async fn channel_playlists_continuation(
|
pub async fn channel_playlists_continuation(
|
||||||
&self,
|
&self,
|
||||||
ctoken: &str,
|
ctoken: &str,
|
||||||
) -> Result<Paginator<ChannelPlaylist>> {
|
) -> Result<Paginator<ChannelPlaylist>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QContinuation {
|
let request_body = QContinuation {
|
||||||
context,
|
context,
|
||||||
|
|
@ -135,7 +135,7 @@ impl RustyPipeQuery {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn channel_info(&self, channel_id: &str) -> Result<Channel<ChannelInfo>> {
|
pub async fn channel_info(&self, channel_id: &str) -> Result<Channel<ChannelInfo>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QChannel {
|
let request_body = QChannel {
|
||||||
context,
|
context,
|
||||||
|
|
@ -160,7 +160,7 @@ impl MapResponse<Channel<Paginator<ChannelVideo>>> for response::Channel {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: crate::model::Language,
|
lang: crate::model::Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Channel<Paginator<ChannelVideo>>>> {
|
) -> Result<MapResult<Channel<Paginator<ChannelVideo>>>, ExtractionError> {
|
||||||
let content = map_channel_content(self.contents, id);
|
let content = map_channel_content(self.contents, id);
|
||||||
let mut warnings = content.warnings;
|
let mut warnings = content.warnings;
|
||||||
let grid = match content.c {
|
let grid = match content.c {
|
||||||
|
|
@ -191,7 +191,7 @@ impl MapResponse<Channel<Paginator<ChannelPlaylist>>> for response::Channel {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Channel<Paginator<ChannelPlaylist>>>> {
|
) -> Result<MapResult<Channel<Paginator<ChannelPlaylist>>>, ExtractionError> {
|
||||||
let content = map_channel_content(self.contents, id);
|
let content = map_channel_content(self.contents, id);
|
||||||
let mut warnings = content.warnings;
|
let mut warnings = content.warnings;
|
||||||
let grid = match content.c {
|
let grid = match content.c {
|
||||||
|
|
@ -222,7 +222,7 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Channel<ChannelInfo>>> {
|
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
|
||||||
let content = map_channel_content(self.contents, id);
|
let content = map_channel_content(self.contents, id);
|
||||||
let mut warnings = content.warnings;
|
let mut warnings = content.warnings;
|
||||||
let meta = match content.c {
|
let meta = match content.c {
|
||||||
|
|
@ -278,11 +278,11 @@ impl MapResponse<Paginator<ChannelVideo>> for response::ChannelCont {
|
||||||
_id: &str,
|
_id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Paginator<ChannelVideo>>> {
|
) -> Result<MapResult<Paginator<ChannelVideo>>, ExtractionError> {
|
||||||
let mut actions = self.on_response_received_actions;
|
let mut actions = self.on_response_received_actions;
|
||||||
let res = some_or_bail!(
|
let res = some_or_bail!(
|
||||||
actions.try_swap_remove(0),
|
actions.try_swap_remove(0),
|
||||||
Err(anyhow!("no received action"))
|
Err(ExtractionError::InvalidData("no received action".into()))
|
||||||
)
|
)
|
||||||
.append_continuation_items_action
|
.append_continuation_items_action
|
||||||
.continuation_items;
|
.continuation_items;
|
||||||
|
|
@ -297,11 +297,11 @@ impl MapResponse<Paginator<ChannelPlaylist>> for response::ChannelCont {
|
||||||
_id: &str,
|
_id: &str,
|
||||||
_lang: Language,
|
_lang: Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Paginator<ChannelPlaylist>>> {
|
) -> Result<MapResult<Paginator<ChannelPlaylist>>, ExtractionError> {
|
||||||
let mut actions = self.on_response_received_actions;
|
let mut actions = self.on_response_received_actions;
|
||||||
let res = some_or_bail!(
|
let res = some_or_bail!(
|
||||||
actions.try_swap_remove(0),
|
actions.try_swap_remove(0),
|
||||||
Err(anyhow!("no received action"))
|
Err(ExtractionError::InvalidData("no received action".into()))
|
||||||
)
|
)
|
||||||
.append_continuation_items_action
|
.append_continuation_items_action
|
||||||
.continuation_items;
|
.continuation_items;
|
||||||
|
|
@ -423,15 +423,14 @@ fn map_channel<T>(
|
||||||
content: T,
|
content: T,
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
) -> Result<Channel<T>> {
|
) -> Result<Channel<T>, ExtractionError> {
|
||||||
let header = header.c4_tabbed_header_renderer;
|
let header = header.c4_tabbed_header_renderer;
|
||||||
|
|
||||||
if header.channel_id != id {
|
if header.channel_id != id {
|
||||||
bail!(
|
return Err(ExtractionError::WrongResult(format!(
|
||||||
"got wrong channel id {}, expected {}",
|
"got wrong channel id {}, expected {}",
|
||||||
header.channel_id,
|
header.channel_id, id
|
||||||
id
|
)));
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Channel {
|
Ok(Channel {
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,15 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
use anyhow::Result;
|
use crate::{
|
||||||
|
error::{Error, ExtractionError},
|
||||||
use crate::{model::ChannelRss, report::Report};
|
model::ChannelRss,
|
||||||
|
report::Report,
|
||||||
|
};
|
||||||
|
|
||||||
use super::{response, RustyPipeQuery};
|
use super::{response, RustyPipeQuery};
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn channel_rss(&self, channel_id: &str) -> Result<ChannelRss> {
|
pub async fn channel_rss(&self, channel_id: &str) -> Result<ChannelRss, Error> {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"https://www.youtube.com/feeds/videos.xml?channel_id={}",
|
"https://www.youtube.com/feeds/videos.xml?channel_id={}",
|
||||||
channel_id
|
channel_id
|
||||||
|
|
@ -41,7 +43,10 @@ impl RustyPipeQuery {
|
||||||
reporter.report(&report);
|
reporter.report(&report);
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(e.into())
|
Err(ExtractionError::InvalidData(
|
||||||
|
format!("could not deserialize xml: {}", e).into(),
|
||||||
|
)
|
||||||
|
.into())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ mod channel_rss;
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Context, Result};
|
|
||||||
use chrono::{DateTime, Duration, Utc};
|
use chrono::{DateTime, Duration, Utc};
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use log::{debug, error, warn};
|
use log::{debug, error, warn};
|
||||||
|
|
@ -27,6 +26,7 @@ use tokio::sync::RwLock;
|
||||||
use crate::{
|
use crate::{
|
||||||
cache::{CacheStorage, FileStorage},
|
cache::{CacheStorage, FileStorage},
|
||||||
deobfuscate::{DeobfData, Deobfuscator},
|
deobfuscate::{DeobfData, Deobfuscator},
|
||||||
|
error::{Error, ExtractionError, Result},
|
||||||
model::{Country, Language},
|
model::{Country, Language},
|
||||||
report::{FileReporter, Level, Report, Reporter},
|
report::{FileReporter, Level, Report, Reporter},
|
||||||
serializer::MapResult,
|
serializer::MapResult,
|
||||||
|
|
@ -166,8 +166,8 @@ pub struct RustyPipe {
|
||||||
|
|
||||||
struct RustyPipeRef {
|
struct RustyPipeRef {
|
||||||
http: Client,
|
http: Client,
|
||||||
storage: Option<Box<dyn CacheStorage + Sync + Send>>,
|
storage: Option<Box<dyn CacheStorage>>,
|
||||||
reporter: Option<Box<dyn Reporter + Sync + Send>>,
|
reporter: Option<Box<dyn Reporter>>,
|
||||||
n_retries: u32,
|
n_retries: u32,
|
||||||
consent_cookie: String,
|
consent_cookie: String,
|
||||||
cache: CacheHolder,
|
cache: CacheHolder,
|
||||||
|
|
@ -183,8 +183,8 @@ struct RustyPipeOpts {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct RustyPipeBuilder {
|
pub struct RustyPipeBuilder {
|
||||||
storage: Option<Box<dyn CacheStorage + Sync + Send>>,
|
storage: Option<Box<dyn CacheStorage>>,
|
||||||
reporter: Option<Box<dyn Reporter + Sync + Send>>,
|
reporter: Option<Box<dyn Reporter>>,
|
||||||
n_retries: u32,
|
n_retries: u32,
|
||||||
user_agent: String,
|
user_agent: String,
|
||||||
default_opts: RustyPipeOpts,
|
default_opts: RustyPipeOpts,
|
||||||
|
|
@ -452,8 +452,11 @@ impl RustyPipe {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Execute the given http request.
|
/// Execute the given http request.
|
||||||
async fn http_request(&self, request: Request) -> Result<Response, reqwest::Error> {
|
async fn http_request(
|
||||||
let mut last_res: Option<Result<Response, reqwest::Error>> = None;
|
&self,
|
||||||
|
request: Request,
|
||||||
|
) -> core::result::Result<Response, reqwest::Error> {
|
||||||
|
let mut last_res = None;
|
||||||
for n in 0..self.inner.n_retries {
|
for n in 0..self.inner.n_retries {
|
||||||
let res = self.inner.http.execute(request.try_clone().unwrap()).await;
|
let res = self.inner.http.execute(request.try_clone().unwrap()).await;
|
||||||
let emsg = match &res {
|
let emsg = match &res {
|
||||||
|
|
@ -509,11 +512,13 @@ impl RustyPipe {
|
||||||
.build()
|
.build()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
.context("Failed to download sw.js")?;
|
|
||||||
|
|
||||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
|
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from(
|
||||||
.ok_or_else(|| anyhow!("Could not find desktop client version in sw.js"))
|
ExtractionError::InvalidData(
|
||||||
|
"Could not find desktop client version in sw.js".into(),
|
||||||
|
),
|
||||||
|
))
|
||||||
};
|
};
|
||||||
|
|
||||||
let from_html = async {
|
let from_html = async {
|
||||||
|
|
@ -525,11 +530,13 @@ impl RustyPipe {
|
||||||
.build()
|
.build()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
.context("Failed to get YT Desktop page")?;
|
|
||||||
|
|
||||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1)
|
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from(
|
||||||
.ok_or_else(|| anyhow!("Could not find desktop client version on html page"))
|
ExtractionError::InvalidData(
|
||||||
|
"Could not find desktop client version in sw.js".into(),
|
||||||
|
),
|
||||||
|
))
|
||||||
};
|
};
|
||||||
|
|
||||||
match from_swjs.await {
|
match from_swjs.await {
|
||||||
|
|
@ -552,11 +559,11 @@ impl RustyPipe {
|
||||||
.build()
|
.build()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
.context("Failed to download sw.js")?;
|
|
||||||
|
|
||||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1)
|
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from(
|
||||||
.ok_or_else(|| anyhow!("Could not find desktop client version in sw.js"))
|
ExtractionError::InvalidData("Could not find music client version in sw.js".into()),
|
||||||
|
))
|
||||||
};
|
};
|
||||||
|
|
||||||
let from_html = async {
|
let from_html = async {
|
||||||
|
|
@ -568,11 +575,13 @@ impl RustyPipe {
|
||||||
.build()
|
.build()
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
)
|
)
|
||||||
.await
|
.await?;
|
||||||
.context("Failed to get YT Desktop page")?;
|
|
||||||
|
|
||||||
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1)
|
util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from(
|
||||||
.ok_or_else(|| anyhow!("Could not find desktop client version on html page"))
|
ExtractionError::InvalidData(
|
||||||
|
"Could not find music client version on html page".into(),
|
||||||
|
),
|
||||||
|
))
|
||||||
};
|
};
|
||||||
|
|
||||||
match from_swjs.await {
|
match from_swjs.await {
|
||||||
|
|
@ -971,7 +980,7 @@ impl RustyPipeQuery {
|
||||||
};
|
};
|
||||||
|
|
||||||
if status.is_client_error() || status.is_server_error() {
|
if status.is_client_error() || status.is_server_error() {
|
||||||
let e = anyhow!("Server responded with error code {}", status);
|
let e = Error::HttpStatus(status.into());
|
||||||
create_report(Level::ERR, Some(e.to_string()), vec![]);
|
create_report(Level::ERR, Some(e.to_string()), vec![]);
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
|
@ -982,12 +991,12 @@ impl RustyPipeQuery {
|
||||||
if !mapres.warnings.is_empty() {
|
if !mapres.warnings.is_empty() {
|
||||||
create_report(
|
create_report(
|
||||||
Level::WRN,
|
Level::WRN,
|
||||||
Some("Warnings during deserialization/mapping".to_owned()),
|
Some(ExtractionError::Warnings.to_string()),
|
||||||
mapres.warnings,
|
mapres.warnings,
|
||||||
);
|
);
|
||||||
|
|
||||||
if self.opts.strict {
|
if self.opts.strict {
|
||||||
bail!("Warnings during deserialization/mapping");
|
return Err(Error::Extraction(ExtractionError::Warnings));
|
||||||
}
|
}
|
||||||
} else if self.opts.report {
|
} else if self.opts.report {
|
||||||
create_report(Level::DBG, None, vec![]);
|
create_report(Level::DBG, None, vec![]);
|
||||||
|
|
@ -995,15 +1004,13 @@ impl RustyPipeQuery {
|
||||||
Ok(mapres.c)
|
Ok(mapres.c)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let emsg = "Could not map reponse";
|
create_report(Level::ERR, Some(e.to_string()), Vec::new());
|
||||||
create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
|
Err(e.into())
|
||||||
Err(e).context(emsg)
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let emsg = "Could not deserialize response";
|
create_report(Level::ERR, Some(e.to_string()), Vec::new());
|
||||||
create_report(Level::ERR, Some(emsg.to_owned()), vec![e.to_string()]);
|
Err(Error::from(ExtractionError::from(e)))
|
||||||
Err(e).context(emsg)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1058,7 +1065,7 @@ trait MapResponse<T> {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
deobf: Option<&Deobfuscator>,
|
deobf: Option<&Deobfuscator>,
|
||||||
) -> Result<MapResult<T>>;
|
) -> core::result::Result<MapResult<T>, crate::error::ExtractionError>;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use anyhow::Result;
|
use crate::error::Result;
|
||||||
|
|
||||||
use crate::model::{
|
use crate::model::{
|
||||||
ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo,
|
ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ use std::{
|
||||||
collections::{BTreeMap, HashMap},
|
collections::{BTreeMap, HashMap},
|
||||||
};
|
};
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
|
||||||
use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
|
use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
|
|
@ -12,6 +11,7 @@ use url::Url;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
|
error::{DeobfError, Error, ExtractionError},
|
||||||
model::{
|
model::{
|
||||||
AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Language, Subtitle,
|
AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Language, Subtitle,
|
||||||
VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream,
|
VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream,
|
||||||
|
|
@ -58,7 +58,11 @@ struct QContentPlaybackContext {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn player(self, video_id: &str, client_type: ClientType) -> Result<VideoPlayer> {
|
pub async fn player(
|
||||||
|
self,
|
||||||
|
video_id: &str,
|
||||||
|
client_type: ClientType,
|
||||||
|
) -> Result<VideoPlayer, Error> {
|
||||||
let q1 = self.clone();
|
let q1 = self.clone();
|
||||||
let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await });
|
let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await });
|
||||||
let q2 = self.client.clone();
|
let q2 = self.client.clone();
|
||||||
|
|
@ -111,7 +115,7 @@ impl MapResponse<VideoPlayer> for response::Player {
|
||||||
id: &str,
|
id: &str,
|
||||||
_lang: Language,
|
_lang: Language,
|
||||||
deobf: Option<&Deobfuscator>,
|
deobf: Option<&Deobfuscator>,
|
||||||
) -> Result<super::MapResult<VideoPlayer>> {
|
) -> Result<super::MapResult<VideoPlayer>, ExtractionError> {
|
||||||
let deobf = deobf.unwrap();
|
let deobf = deobf.unwrap();
|
||||||
let mut warnings = vec![];
|
let mut warnings = vec![];
|
||||||
|
|
||||||
|
|
@ -121,26 +125,36 @@ impl MapResponse<VideoPlayer> for response::Player {
|
||||||
live_streamability.is_some()
|
live_streamability.is_some()
|
||||||
}
|
}
|
||||||
response::player::PlayabilityStatus::Unplayable { reason } => {
|
response::player::PlayabilityStatus::Unplayable { reason } => {
|
||||||
bail!("Video is unplayable. Reason: {}", reason)
|
return Err(ExtractionError::VideoUnavailable("DRM/Geoblock", reason))
|
||||||
}
|
}
|
||||||
response::player::PlayabilityStatus::LoginRequired { reason } => {
|
response::player::PlayabilityStatus::LoginRequired { reason } => {
|
||||||
bail!("Playback requires login. Reason: {}", reason)
|
// reason: "Sign in to confirm your age"
|
||||||
|
if reason.split_whitespace().any(|word| word == "age") {
|
||||||
|
return Err(ExtractionError::VideoAgeRestricted);
|
||||||
|
}
|
||||||
|
return Err(ExtractionError::VideoUnavailable("private video", reason));
|
||||||
}
|
}
|
||||||
response::player::PlayabilityStatus::LiveStreamOffline { reason } => {
|
response::player::PlayabilityStatus::LiveStreamOffline { reason } => {
|
||||||
bail!("Livestream is offline. Reason: {}", reason)
|
return Err(ExtractionError::VideoUnavailable(
|
||||||
|
"offline livestream",
|
||||||
|
reason,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
response::player::PlayabilityStatus::Error { reason } => {
|
response::player::PlayabilityStatus::Error { reason } => {
|
||||||
bail!("Video was deleted. Reason: {}", reason)
|
return Err(ExtractionError::VideoUnavailable(
|
||||||
|
"deletion/censorship",
|
||||||
|
reason,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut streaming_data = some_or_bail!(
|
let mut streaming_data = some_or_bail!(
|
||||||
self.streaming_data,
|
self.streaming_data,
|
||||||
Err(anyhow!("No streaming data was returned"))
|
Err(ExtractionError::InvalidData("no streaming data".into()))
|
||||||
);
|
);
|
||||||
let video_details = some_or_bail!(
|
let video_details = some_or_bail!(
|
||||||
self.video_details,
|
self.video_details,
|
||||||
Err(anyhow!("No video details were returned"))
|
Err(ExtractionError::InvalidData("no video details".into()))
|
||||||
);
|
);
|
||||||
let microformat = self.microformat.map(|m| m.player_microformat_renderer);
|
let microformat = self.microformat.map(|m| m.player_microformat_renderer);
|
||||||
let (publish_date, category, tags, is_family_safe) =
|
let (publish_date, category, tags, is_family_safe) =
|
||||||
|
|
@ -159,11 +173,10 @@ impl MapResponse<VideoPlayer> for response::Player {
|
||||||
});
|
});
|
||||||
|
|
||||||
if video_details.video_id != id {
|
if video_details.video_id != id {
|
||||||
bail!(
|
return Err(ExtractionError::WrongResult(format!(
|
||||||
"got wrong video id {}, expected {}",
|
"video id {}, expected {}",
|
||||||
video_details.video_id,
|
video_details.video_id, id
|
||||||
id
|
)));
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let video_info = VideoPlayerDetails {
|
let video_info = VideoPlayerDetails {
|
||||||
|
|
@ -272,7 +285,7 @@ impl MapResponse<VideoPlayer> for response::Player {
|
||||||
fn cipher_to_url_params(
|
fn cipher_to_url_params(
|
||||||
signature_cipher: &str,
|
signature_cipher: &str,
|
||||||
deobf: &Deobfuscator,
|
deobf: &Deobfuscator,
|
||||||
) -> Result<(String, BTreeMap<String, String>)> {
|
) -> Result<(String, BTreeMap<String, String>), DeobfError> {
|
||||||
let params: HashMap<Cow<str>, Cow<str>> =
|
let params: HashMap<Cow<str>, Cow<str>> =
|
||||||
url::form_urlencoded::parse(signature_cipher.as_bytes()).collect();
|
url::form_urlencoded::parse(signature_cipher.as_bytes()).collect();
|
||||||
|
|
||||||
|
|
@ -281,12 +294,15 @@ fn cipher_to_url_params(
|
||||||
// `sp`: Signature parameter
|
// `sp`: Signature parameter
|
||||||
// `url`: URL that is missing the signature parameter
|
// `url`: URL that is missing the signature parameter
|
||||||
|
|
||||||
let sig = some_or_bail!(params.get("s"), Err(anyhow!("no s param")));
|
let sig = some_or_bail!(params.get("s"), Err(DeobfError::Extraction("s param")));
|
||||||
let sp = some_or_bail!(params.get("sp"), Err(anyhow!("no sp param")));
|
let sp = some_or_bail!(params.get("sp"), Err(DeobfError::Extraction("sp param")));
|
||||||
let raw_url = some_or_bail!(params.get("url"), Err(anyhow!("no url param")));
|
let raw_url = some_or_bail!(
|
||||||
let (url_base, mut url_params) = util::url_to_params(raw_url)?;
|
params.get("url"),
|
||||||
|
Err(DeobfError::Extraction("no url param"))
|
||||||
|
);
|
||||||
|
let (url_base, mut url_params) =
|
||||||
|
util::url_to_params(raw_url).or(Err(DeobfError::Extraction("url params")))?;
|
||||||
|
|
||||||
// println!("sig: {}", sig);
|
|
||||||
let deobf_sig = deobf.deobfuscate_sig(sig)?;
|
let deobf_sig = deobf.deobfuscate_sig(sig)?;
|
||||||
url_params.insert(sp.to_string(), deobf_sig);
|
url_params.insert(sp.to_string(), deobf_sig);
|
||||||
|
|
||||||
|
|
@ -297,7 +313,7 @@ fn deobf_nsig(
|
||||||
url_params: &mut BTreeMap<String, String>,
|
url_params: &mut BTreeMap<String, String>,
|
||||||
deobf: &Deobfuscator,
|
deobf: &Deobfuscator,
|
||||||
last_nsig: &mut [String; 2],
|
last_nsig: &mut [String; 2],
|
||||||
) -> Result<()> {
|
) -> Result<(), DeobfError> {
|
||||||
let nsig: String;
|
let nsig: String;
|
||||||
if let Some(n) = url_params.get("n") {
|
if let Some(n) = url_params.get("n") {
|
||||||
nsig = if n == &last_nsig[0] {
|
nsig = if n == &last_nsig[0] {
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
use std::convert::TryFrom;
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
deobfuscate::Deobfuscator,
|
deobfuscate::Deobfuscator,
|
||||||
|
error::{Error, ExtractionError},
|
||||||
model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo},
|
model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo},
|
||||||
timeago,
|
timeago,
|
||||||
util::{self, TryRemove},
|
util::{self, TryRemove},
|
||||||
|
|
@ -22,7 +22,7 @@ struct QPlaylist {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist> {
|
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QPlaylist {
|
let request_body = QPlaylist {
|
||||||
context,
|
context,
|
||||||
|
|
@ -39,7 +39,10 @@ impl RustyPipeQuery {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn playlist_continuation(self, ctoken: &str) -> Result<Paginator<PlaylistVideo>> {
|
pub async fn playlist_continuation(
|
||||||
|
self,
|
||||||
|
ctoken: &str,
|
||||||
|
) -> Result<Paginator<PlaylistVideo>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QContinuation {
|
let request_body = QContinuation {
|
||||||
context,
|
context,
|
||||||
|
|
@ -63,26 +66,32 @@ impl MapResponse<Playlist> for response::Playlist {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: Language,
|
lang: Language,
|
||||||
_deobf: Option<&Deobfuscator>,
|
_deobf: Option<&Deobfuscator>,
|
||||||
) -> Result<MapResult<Playlist>> {
|
) -> Result<MapResult<Playlist>, ExtractionError> {
|
||||||
// TODO: think about a deserializer that deserializes only first list item
|
// TODO: think about a deserializer that deserializes only first list item
|
||||||
let mut tcbr_contents = self.contents.two_column_browse_results_renderer.contents;
|
let mut tcbr_contents = self.contents.two_column_browse_results_renderer.contents;
|
||||||
let video_items = some_or_bail!(
|
let video_items = some_or_bail!(
|
||||||
some_or_bail!(
|
some_or_bail!(
|
||||||
some_or_bail!(
|
some_or_bail!(
|
||||||
tcbr_contents.try_swap_remove(0),
|
tcbr_contents.try_swap_remove(0),
|
||||||
Err(anyhow!("twoColumnBrowseResultsRenderer empty"))
|
Err(ExtractionError::InvalidData(
|
||||||
|
"twoColumnBrowseResultsRenderer empty".into()
|
||||||
|
))
|
||||||
)
|
)
|
||||||
.tab_renderer
|
.tab_renderer
|
||||||
.content
|
.content
|
||||||
.section_list_renderer
|
.section_list_renderer
|
||||||
.contents
|
.contents
|
||||||
.try_swap_remove(0),
|
.try_swap_remove(0),
|
||||||
Err(anyhow!("sectionListRenderer empty"))
|
Err(ExtractionError::InvalidData(
|
||||||
|
"sectionListRenderer empty".into()
|
||||||
|
))
|
||||||
)
|
)
|
||||||
.item_section_renderer
|
.item_section_renderer
|
||||||
.contents
|
.contents
|
||||||
.try_swap_remove(0),
|
.try_swap_remove(0),
|
||||||
Err(anyhow!("itemSectionRenderer empty"))
|
Err(ExtractionError::InvalidData(
|
||||||
|
"itemSectionRenderer empty".into()
|
||||||
|
))
|
||||||
)
|
)
|
||||||
.playlist_video_list_renderer
|
.playlist_video_list_renderer
|
||||||
.contents;
|
.contents;
|
||||||
|
|
@ -94,7 +103,7 @@ impl MapResponse<Playlist> for response::Playlist {
|
||||||
let mut sidebar_items = sidebar.playlist_sidebar_renderer.items;
|
let mut sidebar_items = sidebar.playlist_sidebar_renderer.items;
|
||||||
let mut primary = some_or_bail!(
|
let mut primary = some_or_bail!(
|
||||||
sidebar_items.try_swap_remove(0),
|
sidebar_items.try_swap_remove(0),
|
||||||
Err(anyhow!("no primary sidebar"))
|
Err(ExtractionError::InvalidData("no primary sidebar".into()))
|
||||||
);
|
);
|
||||||
|
|
||||||
(
|
(
|
||||||
|
|
@ -112,7 +121,7 @@ impl MapResponse<Playlist> for response::Playlist {
|
||||||
None => {
|
None => {
|
||||||
let header_banner = some_or_bail!(
|
let header_banner = some_or_bail!(
|
||||||
self.header.playlist_header_renderer.playlist_header_banner,
|
self.header.playlist_header_renderer.playlist_header_banner,
|
||||||
Err(anyhow!("no thumbnail found"))
|
Err(ExtractionError::InvalidData("no thumbnail found".into()))
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut byline = self.header.playlist_header_renderer.byline;
|
let mut byline = self.header.playlist_header_renderer.byline;
|
||||||
|
|
@ -131,7 +140,7 @@ impl MapResponse<Playlist> for response::Playlist {
|
||||||
Some(_) => {
|
Some(_) => {
|
||||||
ok_or_bail!(
|
ok_or_bail!(
|
||||||
util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text),
|
util::parse_numeric(&self.header.playlist_header_renderer.num_videos_text),
|
||||||
Err(anyhow!("no video count"))
|
Err(ExtractionError::InvalidData("no video count".into()))
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
None => videos.len() as u32,
|
None => videos.len() as u32,
|
||||||
|
|
@ -139,7 +148,10 @@ impl MapResponse<Playlist> for response::Playlist {
|
||||||
|
|
||||||
let playlist_id = self.header.playlist_header_renderer.playlist_id;
|
let playlist_id = self.header.playlist_header_renderer.playlist_id;
|
||||||
if playlist_id != id {
|
if playlist_id != id {
|
||||||
bail!("got wrong playlist id {}, expected {}", playlist_id, id);
|
return Err(ExtractionError::WrongResult(format!(
|
||||||
|
"got wrong playlist id {}, expected {}",
|
||||||
|
playlist_id, id
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let name = self.header.playlist_header_renderer.title;
|
let name = self.header.playlist_header_renderer.title;
|
||||||
|
|
@ -178,11 +190,13 @@ impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
|
||||||
_id: &str,
|
_id: &str,
|
||||||
_lang: Language,
|
_lang: Language,
|
||||||
_deobf: Option<&Deobfuscator>,
|
_deobf: Option<&Deobfuscator>,
|
||||||
) -> Result<MapResult<Paginator<PlaylistVideo>>> {
|
) -> Result<MapResult<Paginator<PlaylistVideo>>, ExtractionError> {
|
||||||
let mut actions = self.on_response_received_actions;
|
let mut actions = self.on_response_received_actions;
|
||||||
let action = some_or_bail!(
|
let action = some_or_bail!(
|
||||||
actions.try_swap_remove(0),
|
actions.try_swap_remove(0),
|
||||||
Err(anyhow!("no continuation action"))
|
Err(ExtractionError::InvalidData(
|
||||||
|
"no continuation action".into()
|
||||||
|
))
|
||||||
);
|
);
|
||||||
|
|
||||||
let (items, ctoken) =
|
let (items, ctoken) =
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
use std::convert::TryFrom;
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
error::{Error, ExtractionError},
|
||||||
model::{
|
model::{
|
||||||
ChannelId, ChannelTag, Chapter, Comment, Language, Paginator, RecommendedVideo,
|
ChannelId, ChannelTag, Chapter, Comment, Language, Paginator, RecommendedVideo,
|
||||||
VideoDetails,
|
VideoDetails,
|
||||||
|
|
@ -30,7 +30,7 @@ struct QVideo {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RustyPipeQuery {
|
impl RustyPipeQuery {
|
||||||
pub async fn video_details(self, video_id: &str) -> Result<VideoDetails> {
|
pub async fn video_details(self, video_id: &str) -> Result<VideoDetails, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QVideo {
|
let request_body = QVideo {
|
||||||
context,
|
context,
|
||||||
|
|
@ -49,7 +49,10 @@ impl RustyPipeQuery {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn video_recommendations(self, ctoken: &str) -> Result<Paginator<RecommendedVideo>> {
|
pub async fn video_recommendations(
|
||||||
|
self,
|
||||||
|
ctoken: &str,
|
||||||
|
) -> Result<Paginator<RecommendedVideo>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QContinuation {
|
let request_body = QContinuation {
|
||||||
context,
|
context,
|
||||||
|
|
@ -66,7 +69,7 @@ impl RustyPipeQuery {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn video_comments(self, ctoken: &str) -> Result<Paginator<Comment>> {
|
pub async fn video_comments(self, ctoken: &str) -> Result<Paginator<Comment>, Error> {
|
||||||
let context = self.get_context(ClientType::Desktop, true).await;
|
let context = self.get_context(ClientType::Desktop, true).await;
|
||||||
let request_body = QContinuation {
|
let request_body = QContinuation {
|
||||||
context,
|
context,
|
||||||
|
|
@ -90,12 +93,15 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
|
||||||
id: &str,
|
id: &str,
|
||||||
lang: crate::model::Language,
|
lang: crate::model::Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<VideoDetails>> {
|
) -> Result<MapResult<VideoDetails>, ExtractionError> {
|
||||||
let mut warnings = Vec::new();
|
let mut warnings = Vec::new();
|
||||||
|
|
||||||
let video_id = self.current_video_endpoint.watch_endpoint.video_id;
|
let video_id = self.current_video_endpoint.watch_endpoint.video_id;
|
||||||
if id != video_id {
|
if id != video_id {
|
||||||
bail!("got wrong playlist id {}, expected {}", video_id, id);
|
return Err(ExtractionError::WrongResult(format!(
|
||||||
|
"got wrong playlist id {}, expected {}",
|
||||||
|
video_id, id
|
||||||
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut primary_results = self
|
let mut primary_results = self
|
||||||
|
|
@ -167,7 +173,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
|
||||||
view_count.video_view_count_renderer.is_live,
|
view_count.video_view_count_renderer.is_live,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
_ => bail!("could not find primary_info"),
|
_ => {
|
||||||
|
return Err(ExtractionError::InvalidData(
|
||||||
|
"could not find primary_info".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let comment_count = comment_count_section.and_then(|s| {
|
let comment_count = comment_count_section.and_then(|s| {
|
||||||
|
|
@ -214,7 +224,11 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
|
||||||
|
|
||||||
(owner.video_owner_renderer, desc, is_ccommons)
|
(owner.video_owner_renderer, desc, is_ccommons)
|
||||||
}
|
}
|
||||||
_ => bail!("could not find secondary_info"),
|
_ => {
|
||||||
|
return Err(ExtractionError::InvalidData(
|
||||||
|
"could not find secondary_info".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let (channel_id, channel_name) = match owner.title {
|
let (channel_id, channel_name) = match owner.title {
|
||||||
|
|
@ -224,9 +238,13 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
|
||||||
browse_id,
|
browse_id,
|
||||||
} => match page_type {
|
} => match page_type {
|
||||||
crate::serializer::text::PageType::Channel => (browse_id, text),
|
crate::serializer::text::PageType::Channel => (browse_id, text),
|
||||||
_ => bail!("invalid channel link type"),
|
_ => {
|
||||||
|
return Err(ExtractionError::InvalidData(
|
||||||
|
"invalid channel link type".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
},
|
},
|
||||||
_ => bail!("invalid channel link"),
|
_ => return Err(ExtractionError::InvalidData("invalid channel link".into())),
|
||||||
};
|
};
|
||||||
|
|
||||||
let recommended = self
|
let recommended = self
|
||||||
|
|
@ -324,11 +342,13 @@ impl MapResponse<Paginator<RecommendedVideo>> for response::VideoRecommendations
|
||||||
_id: &str,
|
_id: &str,
|
||||||
lang: crate::model::Language,
|
lang: crate::model::Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Paginator<RecommendedVideo>>> {
|
) -> Result<MapResult<Paginator<RecommendedVideo>>, ExtractionError> {
|
||||||
let mut endpoints = self.on_response_received_endpoints;
|
let mut endpoints = self.on_response_received_endpoints;
|
||||||
let cont = some_or_bail!(
|
let cont = some_or_bail!(
|
||||||
endpoints.try_swap_remove(0),
|
endpoints.try_swap_remove(0),
|
||||||
Err(anyhow!("no continuation endpoint"))
|
Err(ExtractionError::InvalidData(
|
||||||
|
"no continuation endpoint".into()
|
||||||
|
))
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok(map_recommendations(
|
Ok(map_recommendations(
|
||||||
|
|
@ -344,7 +364,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
|
||||||
_id: &str,
|
_id: &str,
|
||||||
lang: crate::model::Language,
|
lang: crate::model::Language,
|
||||||
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
|
||||||
) -> Result<MapResult<Paginator<Comment>>> {
|
) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
|
||||||
let mut warnings = self.on_response_received_endpoints.warnings;
|
let mut warnings = self.on_response_received_endpoints.warnings;
|
||||||
|
|
||||||
let mut comments = Vec::new();
|
let mut comments = Vec::new();
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
use anyhow::{anyhow, bail, Context, Result};
|
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
|
|
@ -6,7 +5,9 @@ use reqwest::Client;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::result::Result::Ok;
|
use std::result::Result::Ok;
|
||||||
|
|
||||||
use crate::util;
|
use crate::{error::DeobfError, util};
|
||||||
|
|
||||||
|
type Result<T> = core::result::Result<T, DeobfError>;
|
||||||
|
|
||||||
pub struct Deobfuscator {
|
pub struct Deobfuscator {
|
||||||
data: DeobfData,
|
data: DeobfData,
|
||||||
|
|
@ -22,13 +23,8 @@ pub struct DeobfData {
|
||||||
|
|
||||||
impl Deobfuscator {
|
impl Deobfuscator {
|
||||||
pub async fn new(http: Client) -> Result<Self> {
|
pub async fn new(http: Client) -> Result<Self> {
|
||||||
let js_url = get_player_js_url(&http)
|
let js_url = get_player_js_url(&http).await?;
|
||||||
.await
|
let player_js = get_response(&http, &js_url).await?;
|
||||||
.context("Failed to retrieve player.js URL")?;
|
|
||||||
|
|
||||||
let player_js = get_response(&http, &js_url)
|
|
||||||
.await
|
|
||||||
.context("Failed to download player.js")?;
|
|
||||||
|
|
||||||
debug!("Downloaded player.js from {}", js_url);
|
debug!("Downloaded player.js from {}", js_url);
|
||||||
|
|
||||||
|
|
@ -84,7 +80,7 @@ fn get_sig_fn_name(player_js: &str) -> Result<String> {
|
||||||
});
|
});
|
||||||
|
|
||||||
util::get_cg_from_regexes(FUNCTION_PATTERNS.iter(), player_js, 1)
|
util::get_cg_from_regexes(FUNCTION_PATTERNS.iter(), player_js, 1)
|
||||||
.ok_or_else(|| anyhow!("could not find deobf function name"))
|
.ok_or(DeobfError::Extraction("deobf function name"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn caller_function(fn_name: &str) -> String {
|
fn caller_function(fn_name: &str) -> String {
|
||||||
|
|
@ -98,13 +94,13 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
|
||||||
"(".to_owned() + &dfunc_name.replace('$', "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
|
"(".to_owned() + &dfunc_name.replace('$', "\\$") + "=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})";
|
||||||
let function_pattern = ok_or_bail!(
|
let function_pattern = ok_or_bail!(
|
||||||
Regex::new(&function_pattern_str),
|
Regex::new(&function_pattern_str),
|
||||||
Err(anyhow!("could not parse function pattern regex"))
|
Err(DeobfError::Other("could not parse function pattern regex"))
|
||||||
);
|
);
|
||||||
|
|
||||||
let deobfuscate_function = "var ".to_owned()
|
let deobfuscate_function = "var ".to_owned()
|
||||||
+ some_or_bail!(
|
+ some_or_bail!(
|
||||||
function_pattern.captures(player_js).ok().flatten(),
|
function_pattern.captures(player_js).ok().flatten(),
|
||||||
Err(anyhow!("could not find deobf function"))
|
Err(DeobfError::Extraction("deobf function"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -118,7 +114,7 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
|
||||||
.captures(&deobfuscate_function)
|
.captures(&deobfuscate_function)
|
||||||
.ok()
|
.ok()
|
||||||
.flatten(),
|
.flatten(),
|
||||||
Err(anyhow!("could not find helper object name"))
|
Err(DeobfError::Extraction("helper object name"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -128,12 +124,12 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
|
||||||
"(var ".to_owned() + &helper_object_name.replace('$', "\\$") + "=\\{.+?\\}\\};)";
|
"(var ".to_owned() + &helper_object_name.replace('$', "\\$") + "=\\{.+?\\}\\};)";
|
||||||
let helper_pattern = ok_or_bail!(
|
let helper_pattern = ok_or_bail!(
|
||||||
Regex::new(&helper_pattern_str),
|
Regex::new(&helper_pattern_str),
|
||||||
Err(anyhow!("could not parse helper pattern regex"))
|
Err(DeobfError::Other("could not parse helper pattern regex"))
|
||||||
);
|
);
|
||||||
let player_js_nonl = player_js.replace('\n', "");
|
let player_js_nonl = player_js.replace('\n', "");
|
||||||
let helper_object = some_or_bail!(
|
let helper_object = some_or_bail!(
|
||||||
helper_pattern.captures(&player_js_nonl).ok().flatten(),
|
helper_pattern.captures(&player_js_nonl).ok().flatten(),
|
||||||
Err(anyhow!("could not find helper object"))
|
Err(DeobfError::Extraction("helper object"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -143,14 +139,15 @@ fn get_sig_fn(player_js: &str) -> Result<String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn deobfuscate_sig(sig: &str, sig_fn: &str) -> Result<String> {
|
fn deobfuscate_sig(sig: &str, sig_fn: &str) -> Result<String> {
|
||||||
let context = quick_js::Context::new()?;
|
let context =
|
||||||
|
quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?;
|
||||||
context.eval(sig_fn)?;
|
context.eval(sig_fn)?;
|
||||||
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
|
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
|
||||||
|
|
||||||
match res.as_str() {
|
res.as_str().map_or(
|
||||||
Some(res) => Ok(res.to_owned()),
|
Err(DeobfError::Other("sig deobfuscation func returned null")),
|
||||||
None => bail!("deobfuscation func returned null"),
|
|res| Ok(res.to_owned()),
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_nsig_fn_name(player_js: &str) -> Result<String> {
|
fn get_nsig_fn_name(player_js: &str) -> Result<String> {
|
||||||
|
|
@ -161,7 +158,7 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
|
||||||
|
|
||||||
let fname_match = some_or_bail!(
|
let fname_match = some_or_bail!(
|
||||||
FUNCTION_NAME_PATTERN.captures(player_js).ok().flatten(),
|
FUNCTION_NAME_PATTERN.captures(player_js).ok().flatten(),
|
||||||
Err(anyhow!("could not find n_deobf function"))
|
Err(DeobfError::Extraction("n_deobf function"))
|
||||||
);
|
);
|
||||||
|
|
||||||
let function_name = fname_match.get(1).unwrap().as_str();
|
let function_name = fname_match.get(1).unwrap().as_str();
|
||||||
|
|
@ -170,25 +167,29 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String> {
|
||||||
return Ok(function_name.to_owned());
|
return Ok(function_name.to_owned());
|
||||||
}
|
}
|
||||||
|
|
||||||
let array_num = fname_match.get(2).unwrap().as_str().parse::<u32>()?;
|
let array_num = fname_match
|
||||||
|
.get(2)
|
||||||
|
.unwrap()
|
||||||
|
.as_str()
|
||||||
|
.parse::<usize>()
|
||||||
|
.or(Err(DeobfError::Other("could not parse array_num")))?;
|
||||||
let array_pattern_str =
|
let array_pattern_str =
|
||||||
"var ".to_owned() + &fancy_regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];";
|
"var ".to_owned() + &fancy_regex::escape(function_name) + "\\s*=\\s*\\[(.+?)];";
|
||||||
let array_pattern = Regex::new(&array_pattern_str)?;
|
let array_pattern = Regex::new(&array_pattern_str).or(Err(DeobfError::Other(
|
||||||
|
"could not parse helper pattern regex",
|
||||||
|
)))?;
|
||||||
|
|
||||||
let array_str = some_or_bail!(
|
let array_str = some_or_bail!(
|
||||||
array_pattern.captures(player_js).ok().flatten(),
|
array_pattern.captures(player_js).ok().flatten(),
|
||||||
Err(anyhow!("could not find n_deobf array_str"))
|
Err(DeobfError::Extraction("n_deobf array_str"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.as_str();
|
.as_str();
|
||||||
let mut names = array_str.split(',');
|
let mut names = array_str.split(',');
|
||||||
let name = some_or_bail!(
|
let name = some_or_bail!(
|
||||||
names.nth(array_num.try_into()?),
|
names.nth(array_num),
|
||||||
Err(anyhow!(
|
Err(DeobfError::Extraction("n_deobf function name"))
|
||||||
"could not get {}th item from {}",
|
|
||||||
array_num,
|
|
||||||
array_str
|
|
||||||
))
|
|
||||||
);
|
);
|
||||||
Ok(name.to_owned())
|
Ok(name.to_owned())
|
||||||
}
|
}
|
||||||
|
|
@ -239,7 +240,7 @@ fn extract_js_fn(js: &str, name: &str) -> Result<String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
if state != 3 {
|
if state != 3 {
|
||||||
return Err(anyhow!("could not extract js fn"));
|
return Err(DeobfError::Extraction("javascript function"));
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(js[start..end].to_owned())
|
Ok(js[start..end].to_owned())
|
||||||
|
|
@ -255,14 +256,15 @@ fn get_nsig_fn(player_js: &str) -> Result<String> {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn deobfuscate_nsig(sig: &str, nsig_fn: &str) -> Result<String> {
|
fn deobfuscate_nsig(sig: &str, nsig_fn: &str) -> Result<String> {
|
||||||
let context = quick_js::Context::new()?;
|
let context =
|
||||||
|
quick_js::Context::new().or(Err(DeobfError::Other("could not create QuickJS rt")))?;
|
||||||
context.eval(nsig_fn)?;
|
context.eval(nsig_fn)?;
|
||||||
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
|
let res = context.call_function(DEOBFUSCATION_FUNC_NAME, vec![sig])?;
|
||||||
|
|
||||||
match res.as_str() {
|
res.as_str().map_or(
|
||||||
Some(res) => Ok(res.to_owned()),
|
Err(DeobfError::Other("nsig deobfuscation func returned null")),
|
||||||
None => bail!("deobfuscation func returned null"),
|
|res| Ok(res.to_owned()),
|
||||||
}
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_player_js_url(http: &Client) -> Result<String> {
|
async fn get_player_js_url(http: &Client) -> Result<String> {
|
||||||
|
|
@ -278,8 +280,8 @@ async fn get_player_js_url(http: &Client) -> Result<String> {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
});
|
});
|
||||||
let player_hash = some_or_bail!(
|
let player_hash = some_or_bail!(
|
||||||
PLAYER_HASH_PATTERN.captures(&text)?,
|
PLAYER_HASH_PATTERN.captures(&text).ok().flatten(),
|
||||||
Err(anyhow!("could not find player hash"))
|
Err(DeobfError::Extraction("player hash"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -301,8 +303,8 @@ fn get_sts(player_js: &str) -> Result<String> {
|
||||||
Lazy::new(|| Regex::new("signatureTimestamp[=:](\\d+)").unwrap());
|
Lazy::new(|| Regex::new("signatureTimestamp[=:](\\d+)").unwrap());
|
||||||
|
|
||||||
Ok(some_or_bail!(
|
Ok(some_or_bail!(
|
||||||
STS_PATTERN.captures(player_js)?,
|
STS_PATTERN.captures(player_js).ok().flatten(),
|
||||||
Err(anyhow!("could not find sts"))
|
Err(DeobfError::Extraction("sts"))
|
||||||
)
|
)
|
||||||
.get(1)
|
.get(1)
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@
|
||||||
|
|
||||||
use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf};
|
use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf};
|
||||||
|
|
||||||
use anyhow::{anyhow, bail, Result};
|
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
use indicatif::ProgressBar;
|
use indicatif::ProgressBar;
|
||||||
|
|
@ -17,10 +16,13 @@ use tokio::{
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
|
error::DownloadError,
|
||||||
model::{stream_filter::Filter, AudioCodec, FileFormat, VideoCodec, VideoPlayer},
|
model::{stream_filter::Filter, AudioCodec, FileFormat, VideoCodec, VideoPlayer},
|
||||||
util,
|
util,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type Result<T> = core::result::Result<T, DownloadError>;
|
||||||
|
|
||||||
const CHUNK_SIZE_MIN: u64 = 9000000;
|
const CHUNK_SIZE_MIN: u64 = 9000000;
|
||||||
const CHUNK_SIZE_MAX: u64 = 10000000;
|
const CHUNK_SIZE_MAX: u64 = 10000000;
|
||||||
|
|
||||||
|
|
@ -44,15 +46,32 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
|
||||||
|
|
||||||
let captures = some_or_bail!(
|
let captures = some_or_bail!(
|
||||||
PATTERN.captures(cr_header).ok().flatten(),
|
PATTERN.captures(cr_header).ok().flatten(),
|
||||||
Err(anyhow!(
|
Err(DownloadError::Progressive(
|
||||||
"Content-Range header '{}' does not match pattern.",
|
format!(
|
||||||
cr_header
|
"Content-Range header '{}' does not match pattern",
|
||||||
|
cr_header
|
||||||
|
)
|
||||||
|
.into()
|
||||||
))
|
))
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
captures.get(2).unwrap().as_str().parse()?,
|
captures
|
||||||
captures.get(3).unwrap().as_str().parse()?,
|
.get(2)
|
||||||
|
.unwrap()
|
||||||
|
.as_str()
|
||||||
|
.parse()
|
||||||
|
.or(Err(DownloadError::Progressive(
|
||||||
|
"could not parse range header number".into(),
|
||||||
|
)))?,
|
||||||
|
captures
|
||||||
|
.get(3)
|
||||||
|
.unwrap()
|
||||||
|
.as_str()
|
||||||
|
.parse()
|
||||||
|
.or(Err(DownloadError::Progressive(
|
||||||
|
"could not parse range header number".into(),
|
||||||
|
)))?,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -76,7 +95,8 @@ async fn download_single_file<P: Into<PathBuf>>(
|
||||||
let mut size: Option<u64> = None;
|
let mut size: Option<u64> = None;
|
||||||
|
|
||||||
// If the url is from googlevideo, extract file size from clen parameter
|
// If the url is from googlevideo, extract file size from clen parameter
|
||||||
let (url_base, url_params) = util::url_to_params(url)?;
|
let (url_base, url_params) =
|
||||||
|
util::url_to_params(url).or_else(|e| Err(DownloadError::Other(e.to_string().into())))?;
|
||||||
let is_gvideo = url_base.ends_with(".googlevideo.com/videoplayback");
|
let is_gvideo = url_base.ends_with(".googlevideo.com/videoplayback");
|
||||||
if is_gvideo {
|
if is_gvideo {
|
||||||
size = url_params.get("clen").and_then(|s| s.parse::<u64>().ok());
|
size = url_params.get("clen").and_then(|s| s.parse::<u64>().ok());
|
||||||
|
|
@ -95,9 +115,14 @@ async fn download_single_file<P: Into<PathBuf>>(
|
||||||
|
|
||||||
let cr_header = some_or_bail!(
|
let cr_header = some_or_bail!(
|
||||||
res.headers().get(header::CONTENT_RANGE),
|
res.headers().get(header::CONTENT_RANGE),
|
||||||
Err(anyhow!("Did not get Content-Range header"))
|
Err(DownloadError::Progressive(
|
||||||
|
"Did not get Content-Range header".into()
|
||||||
|
))
|
||||||
)
|
)
|
||||||
.to_str()?;
|
.to_str()
|
||||||
|
.or(Err(DownloadError::Progressive(
|
||||||
|
"could not convert Content-Range header to string".into(),
|
||||||
|
)))?;
|
||||||
|
|
||||||
let (_, original_size) = parse_cr_header(cr_header)?;
|
let (_, original_size) = parse_cr_header(cr_header)?;
|
||||||
|
|
||||||
|
|
@ -117,9 +142,12 @@ async fn download_single_file<P: Into<PathBuf>>(
|
||||||
}
|
}
|
||||||
Ordering::Greater => {
|
Ordering::Greater => {
|
||||||
// WTF?
|
// WTF?
|
||||||
return Err(anyhow!(
|
return Err(DownloadError::Other(
|
||||||
"Already downloaded file {} is larger than original",
|
format!(
|
||||||
output_path_tmp.to_str().unwrap_or_default()
|
"Already downloaded file {} is larger than original",
|
||||||
|
output_path_tmp.to_str().unwrap_or_default()
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -174,9 +202,14 @@ async fn download_chunks_by_header(
|
||||||
// Content-Range: bytes 0-100/451368980
|
// Content-Range: bytes 0-100/451368980
|
||||||
let cr_header = some_or_bail!(
|
let cr_header = some_or_bail!(
|
||||||
res.headers().get(header::CONTENT_RANGE),
|
res.headers().get(header::CONTENT_RANGE),
|
||||||
Err(anyhow!("Did not get Content-Range header"))
|
Err(DownloadError::Progressive(
|
||||||
|
"Did not get Content-Range header".into()
|
||||||
|
))
|
||||||
)
|
)
|
||||||
.to_str()?;
|
.to_str()
|
||||||
|
.or(Err(DownloadError::Progressive(
|
||||||
|
"could not convert Content-Range header to string".into(),
|
||||||
|
)))?;
|
||||||
|
|
||||||
let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?;
|
let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?;
|
||||||
|
|
||||||
|
|
@ -279,7 +312,7 @@ pub async fn download_video(
|
||||||
let (video, audio) = player_data.select_video_audio_stream(filter);
|
let (video, audio) = player_data.select_video_audio_stream(filter);
|
||||||
|
|
||||||
if video.is_none() && audio.is_none() {
|
if video.is_none() && audio.is_none() {
|
||||||
return Err(anyhow!("no stream found"));
|
return Err(DownloadError::Input("no stream found".into()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let format = output_format.unwrap_or(
|
let format = output_format.unwrap_or(
|
||||||
|
|
@ -287,7 +320,9 @@ pub async fn download_video(
|
||||||
Some(_) => "mp4",
|
Some(_) => "mp4",
|
||||||
None => match audio {
|
None => match audio {
|
||||||
Some(audio) => match audio.codec {
|
Some(audio) => match audio.codec {
|
||||||
AudioCodec::Unknown => return Err(anyhow!("unknown audio codec")),
|
AudioCodec::Unknown => {
|
||||||
|
return Err(DownloadError::Input("unknown audio codec".into()))
|
||||||
|
}
|
||||||
AudioCodec::Mp4a => "m4a",
|
AudioCodec::Mp4a => "m4a",
|
||||||
AudioCodec::Opus => "opus",
|
AudioCodec::Opus => "opus",
|
||||||
},
|
},
|
||||||
|
|
@ -302,7 +337,9 @@ pub async fn download_video(
|
||||||
// If the downloaded video already exists, only error if the download path was
|
// If the downloaded video already exists, only error if the download path was
|
||||||
// chosen explicitly.
|
// chosen explicitly.
|
||||||
if output_fname_set {
|
if output_fname_set {
|
||||||
bail!("File {} already exists", output_path.to_string_lossy());
|
return Err(DownloadError::Input(
|
||||||
|
format!("File {} already exists", output_path.to_string_lossy()).into(),
|
||||||
|
))?;
|
||||||
} else {
|
} else {
|
||||||
info!(
|
info!(
|
||||||
"Downloaded video {} already exists",
|
"Downloaded video {} already exists",
|
||||||
|
|
@ -366,7 +403,7 @@ pub async fn download_video(
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.collect::<Result<_, _>>()?;
|
.collect::<core::result::Result<_, _>>()?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -423,10 +460,13 @@ async fn convert_streams<P: Into<PathBuf>>(
|
||||||
let res = Command::new(ffmpeg).args(args).output().await?;
|
let res = Command::new(ffmpeg).args(args).output().await?;
|
||||||
|
|
||||||
if !res.status.success() {
|
if !res.status.success() {
|
||||||
bail!(
|
return Err(DownloadError::Ffmpeg(
|
||||||
"ffmpeg error: {}",
|
format!(
|
||||||
std::str::from_utf8(&res.stderr).unwrap_or_default()
|
"ffmpeg error: {}",
|
||||||
)
|
std::str::from_utf8(&res.stderr).unwrap_or_default()
|
||||||
|
)
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
91
src/error.rs
Normal file
91
src/error.rs
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
use std::borrow::Cow;
|
||||||
|
|
||||||
|
pub(crate) type Result<T> = core::result::Result<T, Error>;
|
||||||
|
|
||||||
|
/// Custom error type for the RustyPipe library
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
#[non_exhaustive]
|
||||||
|
pub enum Error {
|
||||||
|
/// Error extracting content from YouTube
|
||||||
|
#[error("extraction error: {0}")]
|
||||||
|
Extraction(#[from] ExtractionError),
|
||||||
|
/// Error from the deobfuscater
|
||||||
|
#[error("deobfuscator error: {0}")]
|
||||||
|
Deobfuscation(#[from] DeobfError),
|
||||||
|
/// Error from the video downloader
|
||||||
|
#[error("download error: {0}")]
|
||||||
|
Download(#[from] DownloadError),
|
||||||
|
/// File IO error
|
||||||
|
#[error(transparent)]
|
||||||
|
Io(#[from] std::io::Error),
|
||||||
|
/// Error from the HTTP client
|
||||||
|
#[error("http error: {0}")]
|
||||||
|
Http(#[from] reqwest::Error),
|
||||||
|
#[error("http status code: {0}")]
|
||||||
|
HttpStatus(u16),
|
||||||
|
#[error("error: {0}")]
|
||||||
|
Other(Cow<'static, str>),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Error that occurred during the initialization
|
||||||
|
/// or use of the YouTube URL signature deobfuscator.
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
#[non_exhaustive]
|
||||||
|
pub enum DeobfError {
|
||||||
|
/// Error from the HTTP client
|
||||||
|
#[error("http error: {0}")]
|
||||||
|
Http(#[from] reqwest::Error),
|
||||||
|
/// Error during JavaScript execution
|
||||||
|
#[error("js execution error: {0}")]
|
||||||
|
JavaScript(#[from] quick_js::ExecutionError),
|
||||||
|
#[error("js parsing: {0}")]
|
||||||
|
JsParser(#[from] ress::error::Error),
|
||||||
|
/// Could not extract certain data
|
||||||
|
#[error("could not extract {0}")]
|
||||||
|
Extraction(&'static str),
|
||||||
|
#[error("error: {0}")]
|
||||||
|
Other(&'static str),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Error from the video downloader
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
#[non_exhaustive]
|
||||||
|
pub enum DownloadError {
|
||||||
|
/// Error from the HTTP client
|
||||||
|
#[error("http error: {0}")]
|
||||||
|
Http(#[from] reqwest::Error),
|
||||||
|
/// File IO error
|
||||||
|
#[error(transparent)]
|
||||||
|
Io(#[from] std::io::Error),
|
||||||
|
#[error("FFmpeg error: {0}")]
|
||||||
|
Ffmpeg(Cow<'static, str>),
|
||||||
|
#[error("Progressive download error: {0}")]
|
||||||
|
Progressive(Cow<'static, str>),
|
||||||
|
#[error("input error: {0}")]
|
||||||
|
Input(Cow<'static, str>),
|
||||||
|
#[error("error: {0}")]
|
||||||
|
Other(Cow<'static, str>),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Error extracting content from YouTube
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
#[non_exhaustive]
|
||||||
|
pub enum ExtractionError {
|
||||||
|
#[error("Video cant be played because of {0}. Reason (from YT): {1}")]
|
||||||
|
VideoUnavailable(&'static str, String),
|
||||||
|
#[error("Video is age restricted")]
|
||||||
|
VideoAgeRestricted,
|
||||||
|
#[error("deserialization error: {0}")]
|
||||||
|
Deserialization(#[from] serde_json::Error),
|
||||||
|
#[error("got invalid data from YT: {0}")]
|
||||||
|
InvalidData(Cow<'static, str>),
|
||||||
|
#[error("got wrong result from YT: {0}")]
|
||||||
|
WrongResult(String),
|
||||||
|
#[error("Warnings during deserialization/mapping")]
|
||||||
|
Warnings,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Internal error
|
||||||
|
#[derive(thiserror::Error, Debug)]
|
||||||
|
#[error("mapping error: {0}")]
|
||||||
|
pub struct MappingError(pub(crate) Cow<'static, str>);
|
||||||
|
|
@ -17,6 +17,7 @@ mod util;
|
||||||
pub mod cache;
|
pub mod cache;
|
||||||
pub mod client;
|
pub mod client;
|
||||||
pub mod download;
|
pub mod download;
|
||||||
|
pub mod error;
|
||||||
pub mod model;
|
pub mod model;
|
||||||
pub mod report;
|
pub mod report;
|
||||||
pub mod timeago;
|
pub mod timeago;
|
||||||
|
|
|
||||||
|
|
@ -6,12 +6,12 @@ use std::{
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
};
|
};
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use chrono::{DateTime, Local};
|
use chrono::{DateTime, Local};
|
||||||
use log::error;
|
use log::error;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::deobfuscate::DeobfData;
|
use crate::deobfuscate::DeobfData;
|
||||||
|
use crate::error::{Error, Result};
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
#[non_exhaustive]
|
#[non_exhaustive]
|
||||||
|
|
@ -81,7 +81,7 @@ impl Default for Info {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub trait Reporter {
|
pub trait Reporter: Sync + Send {
|
||||||
fn report(&self, report: &Report);
|
fn report(&self, report: &Report);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -98,7 +98,11 @@ impl FileReporter {
|
||||||
|
|
||||||
fn _report(&self, report: &Report) -> Result<()> {
|
fn _report(&self, report: &Report) -> Result<()> {
|
||||||
let report_path = get_report_path(&self.path, report, "json")?;
|
let report_path = get_report_path(&self.path, report, "json")?;
|
||||||
serde_json::to_writer_pretty(&File::create(report_path)?, &report)?;
|
serde_json::to_writer_pretty(&File::create(report_path)?, &report).or_else(|e| {
|
||||||
|
Err(Error::Other(
|
||||||
|
format!("could not serialize report. err: {}", e).into(),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,11 @@
|
||||||
use std::convert::TryFrom;
|
use std::convert::TryFrom;
|
||||||
|
|
||||||
use anyhow::anyhow;
|
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
use serde::{Deserialize, Deserializer};
|
use serde::{Deserialize, Deserializer};
|
||||||
use serde_with::{serde_as, DefaultOnError, DeserializeAs};
|
use serde_with::{serde_as, DefaultOnError, DeserializeAs};
|
||||||
|
|
||||||
use crate::util;
|
use crate::{error::MappingError, util};
|
||||||
|
|
||||||
/// # Text
|
/// # Text
|
||||||
///
|
///
|
||||||
|
|
@ -366,7 +365,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TryFrom<TextComponent> for crate::model::ChannelId {
|
impl TryFrom<TextComponent> for crate::model::ChannelId {
|
||||||
type Error = anyhow::Error;
|
type Error = MappingError;
|
||||||
|
|
||||||
fn try_from(value: TextComponent) -> Result<Self, Self::Error> {
|
fn try_from(value: TextComponent) -> Result<Self, Self::Error> {
|
||||||
match value {
|
match value {
|
||||||
|
|
@ -379,9 +378,9 @@ impl TryFrom<TextComponent> for crate::model::ChannelId {
|
||||||
id: browse_id,
|
id: browse_id,
|
||||||
name: text,
|
name: text,
|
||||||
}),
|
}),
|
||||||
_ => Err(anyhow!("invalid channel link type")),
|
_ => Err(MappingError("invalid channel link type".into())),
|
||||||
},
|
},
|
||||||
_ => Err(anyhow!("invalid channel link")),
|
_ => Err(MappingError("invalid channel link".into())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
11
src/util.rs
11
src/util.rs
|
|
@ -1,12 +1,11 @@
|
||||||
use std::{borrow::Borrow, collections::BTreeMap, str::FromStr};
|
use std::{borrow::Borrow, collections::BTreeMap, str::FromStr};
|
||||||
|
|
||||||
use anyhow::Result;
|
|
||||||
use fancy_regex::Regex;
|
use fancy_regex::Regex;
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::Lazy;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use crate::{dictionary, model::Language};
|
use crate::{dictionary, error::Error, error::Result, model::Language};
|
||||||
|
|
||||||
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
|
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
|
||||||
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
|
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
|
||||||
|
|
@ -44,7 +43,11 @@ pub fn generate_content_playback_nonce() -> String {
|
||||||
///
|
///
|
||||||
/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}`
|
/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}`
|
||||||
pub fn url_to_params(url: &str) -> Result<(String, BTreeMap<String, String>)> {
|
pub fn url_to_params(url: &str) -> Result<(String, BTreeMap<String, String>)> {
|
||||||
let mut parsed_url = Url::parse(url)?;
|
let mut parsed_url = Url::parse(url).or_else(|e| {
|
||||||
|
Err(Error::Other(
|
||||||
|
format!("could not parse url `{}` err: {}", url, e).into(),
|
||||||
|
))
|
||||||
|
})?;
|
||||||
let url_params: BTreeMap<String, String> = parsed_url
|
let url_params: BTreeMap<String, String> = parsed_url
|
||||||
.query_pairs()
|
.query_pairs()
|
||||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||||
|
|
@ -56,7 +59,7 @@ pub fn url_to_params(url: &str) -> Result<(String, BTreeMap<String, String>)> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a string after removing all non-numeric characters
|
/// Parse a string after removing all non-numeric characters
|
||||||
pub fn parse_numeric<F>(string: &str) -> Result<F, F::Err>
|
pub fn parse_numeric<F>(string: &str) -> core::result::Result<F, F::Err>
|
||||||
where
|
where
|
||||||
F: FromStr,
|
F: FromStr,
|
||||||
{
|
{
|
||||||
|
|
|
||||||
Reference in a new issue