From e063c048217f5b68733ad38d006959d4975e46c7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 1 Dec 2022 00:25:47 +0100 Subject: [PATCH] refactor!: move downloader to seperate crate --- Cargo.toml | 7 +-- cli/Cargo.toml | 13 +++-- cli/src/main.rs | 2 +- downloader/Cargo.toml | 31 ++++++++++++ src/download.rs => downloader/src/lib.rs | 60 +++++------------------- downloader/src/util.rs | 42 +++++++++++++++++ src/error.rs | 23 --------- src/lib.rs | 1 - 8 files changed, 96 insertions(+), 83 deletions(-) create mode 100644 downloader/Cargo.toml rename src/download.rs => downloader/src/lib.rs (91%) create mode 100644 downloader/src/util.rs diff --git a/Cargo.toml b/Cargo.toml index ea67285..1f2c729 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["youtube", "video", "music"] include = ["/src", "README.md", "LICENSE", "!snapshots"] [workspace] -members = [".", "codegen", "cli"] +members = [".", "codegen", "downloader", "cli"] [features] default = ["default-tls"] @@ -34,9 +34,8 @@ reqwest = { version = "0.11.11", default-features = false, features = [ "json", "gzip", "brotli", - "stream", ] } -tokio = { version = "1.20.0", features = ["macros", "time", "fs", "process"] } +tokio = { version = "1.20.0", features = ["macros", "time"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.82" serde_with = { version = "2.0.0", features = ["json"] } @@ -47,8 +46,6 @@ time = { version = "0.3.15", features = [ "serde-well-known", ] } futures = "0.3.21" -indicatif = "0.17.0" -filenamify = "0.1.0" ress = "0.11.4" phf = "0.11.1" base64 = "0.13.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 23b8ed6..53f895e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -4,11 +4,16 @@ version = "0.1.0" edition = "2021" [dependencies] -rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]} -reqwest = {version = "0.11.11", default_features = false} -tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]} +rustypipe = { path = "../", default_features = false, features = [ + "rustls-tls-native-roots", +] } +rustypipe-downloader = { path = "../downloader", default_features = false, features = [ + "rustls-tls-native-roots", +] } +reqwest = { version = "0.11.11", default_features = false } +tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] } indicatif = "0.17.0" futures = "0.3.21" anyhow = "1.0" clap = { version = "3.2.16", features = ["derive"] } -env_logger = "0.9.0" +env_logger = "0.10.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index 0acac75..f6a3dc3 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -69,7 +69,7 @@ async fn download_single_video( } } - rustypipe::download::download_video( + rustypipe_downloader::download_video( &player_data, output_dir, output_fname, diff --git a/downloader/Cargo.toml b/downloader/Cargo.toml new file mode 100644 index 0000000..9f60764 --- /dev/null +++ b/downloader/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "rustypipe-downloader" +version = "0.1.0" +edition = "2021" + +[features] +# Reqwest TLS +default-tls = ["reqwest/default-tls", "rustypipe/default-tls"] +rustls-tls-webpki-roots = [ + "reqwest/rustls-tls-webpki-roots", + "rustypipe/rustls-tls-webpki-roots", +] +rustls-tls-native-roots = [ + "reqwest/rustls-tls-native-roots", + "rustypipe/rustls-tls-native-roots", +] + +[dependencies] +rustypipe = { path = "..", default-features = false } +once_cell = "1.12.0" +regex = "1.6.0" +thiserror = "1.0.36" +futures = "0.3.21" +indicatif = "0.17.0" +filenamify = "0.1.0" +log = "0.4.17" +reqwest = { version = "0.11.11", default-features = false, features = [ + "stream", +] } +rand = "0.8.5" +tokio = { version = "1.20.0", features = ["macros", "fs", "process"] } diff --git a/src/download.rs b/downloader/src/lib.rs similarity index 91% rename from src/download.rs rename to downloader/src/lib.rs index 238e9ab..34adc42 100644 --- a/src/download.rs +++ b/downloader/src/lib.rs @@ -1,26 +1,27 @@ -//! YouTube audio/video downloader +//! # YouTube audio/video downloader + +mod util; use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration}; -use fancy_regex::Regex; use futures::stream::{self, StreamExt}; use indicatif::{ProgressBar, ProgressStyle}; use log::{debug, info}; use once_cell::sync::Lazy; use rand::Rng; +use regex::Regex; use reqwest::{header, Client}; +use rustypipe::{ + model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, + param::StreamFilter, +}; use tokio::{ fs::{self, File}, io::AsyncWriteExt, process::Command, }; -use crate::{ - error::DownloadError, - model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, - param::StreamFilter, - util, -}; +use util::DownloadError; type Result = core::result::Result; @@ -45,7 +46,7 @@ fn get_download_range(offset: u64, size: Option) -> Range { fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { static PATTERN: Lazy = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap()); - let captures = PATTERN.captures(cr_header).ok().flatten().ok_or_else(|| { + let captures = PATTERN.captures(cr_header).ok_or_else(|| { DownloadError::Progressive( format!( "Content-Range header '{}' does not match pattern", @@ -317,11 +318,9 @@ pub async fn download_video( Some(_) => "mp4", None => match audio { Some(audio) => match audio.codec { - AudioCodec::Unknown => { - return Err(DownloadError::Input("unknown audio codec".into())) - } AudioCodec::Mp4a => "m4a", AudioCodec::Opus => "opus", + _ => return Err(DownloadError::Input("unknown audio codec".into())), }, None => unreachable!(), }, @@ -473,40 +472,3 @@ async fn convert_streams>( } Ok(()) } - -/* -#[cfg(test)] -mod tests { - use crate::client::RustyTube; - - use super::*; - use indicatif::{ProgressDrawTarget, ProgressStyle}; - use reqwest::ClientBuilder; - - // #[test_log::test(tokio::test)] - #[tokio::test] - async fn t_download_video() { - let http = ClientBuilder::new() - .user_agent( - "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0", - ) - .gzip(true) - .brotli(true) - .build() - .expect("unable to build the HTTP client"); - - // Indicatif setup - let pb = ProgressBar::new(0); - - let rt = RustyTube::new(); - let player_data = rt - .get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop) - .await - .unwrap(); - - // download_video(&player_data, "tmp", "INVU", Some(1080), "ffmpeg", http, pb) - // .await - // .unwrap(); - } -} -*/ diff --git a/downloader/src/util.rs b/downloader/src/util.rs new file mode 100644 index 0000000..b6b6719 --- /dev/null +++ b/downloader/src/util.rs @@ -0,0 +1,42 @@ +use std::{borrow::Cow, collections::BTreeMap}; + +use reqwest::Url; + +/// Error from the video downloader +#[derive(thiserror::Error, Debug)] +#[non_exhaustive] +pub enum DownloadError { + /// Error from the HTTP client + #[error("http error: {0}")] + Http(#[from] reqwest::Error), + /// File IO error + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("FFmpeg error: {0}")] + Ffmpeg(Cow<'static, str>), + #[error("Progressive download error: {0}")] + Progressive(Cow<'static, str>), + #[error("input error: {0}")] + Input(Cow<'static, str>), + #[error("error: {0}")] + Other(Cow<'static, str>), +} + +/// Split an URL into its base string and parameter map +/// +/// Example: +/// +/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` +pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap), DownloadError> { + let mut parsed_url = Url::parse(url).map_err(|e| { + DownloadError::Other(format!("could not parse url `{}` err: {}", url, e).into()) + })?; + let url_params: BTreeMap = parsed_url + .query_pairs() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + parsed_url.set_query(None); + + Ok((parsed_url, url_params)) +} diff --git a/src/error.rs b/src/error.rs index ced5358..1ce1641 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,9 +10,6 @@ pub enum Error { /// Error from the deobfuscater #[error("deobfuscator error: {0}")] Deobfuscation(#[from] DeobfError), - /// Error from the video downloader - #[error("download error: {0}")] - Download(#[from] DownloadError), /// File IO error #[error(transparent)] Io(#[from] std::io::Error), @@ -45,26 +42,6 @@ pub enum DeobfError { Other(&'static str), } -/// Error from the video downloader -#[derive(thiserror::Error, Debug)] -#[non_exhaustive] -pub enum DownloadError { - /// Error from the HTTP client - #[error("http error: {0}")] - Http(#[from] reqwest::Error), - /// File IO error - #[error(transparent)] - Io(#[from] std::io::Error), - #[error("FFmpeg error: {0}")] - Ffmpeg(Cow<'static, str>), - #[error("Progressive download error: {0}")] - Progressive(Cow<'static, str>), - #[error("input error: {0}")] - Input(Cow<'static, str>), - #[error("error: {0}")] - Other(Cow<'static, str>), -} - /// Error extracting content from YouTube #[derive(thiserror::Error, Debug)] #[non_exhaustive] diff --git a/src/lib.rs b/src/lib.rs index f06ca4d..f406da5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,6 @@ mod util; pub mod cache; pub mod client; -pub mod download; pub mod error; pub mod model; pub mod param;