refactor!: move downloader to seperate crate

Cargo.toml

@@ -10,7 +10,7 @@ keywords = ["youtube", "video", "music"]
 include = ["/src", "README.md", "LICENSE", "!snapshots"]
 
 [workspace]
-members = [".", "codegen", "cli"]
+members = [".", "codegen", "downloader", "cli"]
 
 [features]
 default = ["default-tls"]
@@ -34,9 +34,8 @@ reqwest = { version = "0.11.11", default-features = false, features = [
     "json",
     "gzip",
     "brotli",
-    "stream",
 ] }
-tokio = { version = "1.20.0", features = ["macros", "time", "fs", "process"] }
+tokio = { version = "1.20.0", features = ["macros", "time"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.82"
 serde_with = { version = "2.0.0", features = ["json"] }
@@ -47,8 +46,6 @@ time = { version = "0.3.15", features = [
     "serde-well-known",
 ] }
 futures = "0.3.21"
-indicatif = "0.17.0"
-filenamify = "0.1.0"
 ress = "0.11.4"
 phf = "0.11.1"
 base64 = "0.13.0"

cli/Cargo.toml

@@ -4,11 +4,16 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-rustypipe = {path = "../", default_features = false, features = ["rustls-tls-native-roots"]}
+rustypipe = { path = "../", default_features = false, features = [
-reqwest = {version = "0.11.11", default_features = false}
+    "rustls-tls-native-roots",
-tokio = {version = "1.20.0", features = ["macros", "rt-multi-thread"]}
+] }
+rustypipe-downloader = { path = "../downloader", default_features = false, features = [
+    "rustls-tls-native-roots",
+] }
+reqwest = { version = "0.11.11", default_features = false }
+tokio = { version = "1.20.0", features = ["macros", "rt-multi-thread"] }
 indicatif = "0.17.0"
 futures = "0.3.21"
 anyhow = "1.0"
 clap = { version = "3.2.16", features = ["derive"] }
-env_logger = "0.9.0"
+env_logger = "0.10.0"

cli/src/main.rs

@@ -69,7 +69,7 @@ async fn download_single_video(
             }
         }
 
-        rustypipe::download::download_video(
+        rustypipe_downloader::download_video(
             &player_data,
             output_dir,
             output_fname,

downloader/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "rustypipe-downloader"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+# Reqwest TLS
+default-tls = ["reqwest/default-tls", "rustypipe/default-tls"]
+rustls-tls-webpki-roots = [
+    "reqwest/rustls-tls-webpki-roots",
+    "rustypipe/rustls-tls-webpki-roots",
+]
+rustls-tls-native-roots = [
+    "reqwest/rustls-tls-native-roots",
+    "rustypipe/rustls-tls-native-roots",
+]
+
+[dependencies]
+rustypipe = { path = "..", default-features = false }
+once_cell = "1.12.0"
+regex = "1.6.0"
+thiserror = "1.0.36"
+futures = "0.3.21"
+indicatif = "0.17.0"
+filenamify = "0.1.0"
+log = "0.4.17"
+reqwest = { version = "0.11.11", default-features = false, features = [
+    "stream",
+] }
+rand = "0.8.5"
+tokio = { version = "1.20.0", features = ["macros", "fs", "process"] }

downloader/src/lib.rs

@@ -1,26 +1,27 @@
-//! YouTube audio/video downloader
+//! # YouTube audio/video downloader
+
+mod util;
 
 use std::{borrow::Cow, cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf, time::Duration};
 
-use fancy_regex::Regex;
 use futures::stream::{self, StreamExt};
 use indicatif::{ProgressBar, ProgressStyle};
 use log::{debug, info};
 use once_cell::sync::Lazy;
 use rand::Rng;
+use regex::Regex;
 use reqwest::{header, Client};
+use rustypipe::{
+    model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer},
+    param::StreamFilter,
+};
 use tokio::{
     fs::{self, File},
     io::AsyncWriteExt,
     process::Command,
 };
 
-use crate::{
+use util::DownloadError;
-    error::DownloadError,
-    model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer},
-    param::StreamFilter,
-    util,
-};
 
 type Result<T> = core::result::Result<T, DownloadError>;
 
@@ -45,7 +46,7 @@ fn get_download_range(offset: u64, size: Option<u64>) -> Range<u64> {
 fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
     static PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"bytes (\d+)-(\d+)/(\d+)"#).unwrap());
 
-    let captures = PATTERN.captures(cr_header).ok().flatten().ok_or_else(|| {
+    let captures = PATTERN.captures(cr_header).ok_or_else(|| {
         DownloadError::Progressive(
             format!(
                 "Content-Range header '{}' does not match pattern",
@@ -317,11 +318,9 @@ pub async fn download_video(
             Some(_) => "mp4",
             None => match audio {
                 Some(audio) => match audio.codec {
-                    AudioCodec::Unknown => {
-                        return Err(DownloadError::Input("unknown audio codec".into()))
-                    }
                     AudioCodec::Mp4a => "m4a",
                     AudioCodec::Opus => "opus",
+                    _ => return Err(DownloadError::Input("unknown audio codec".into())),
                 },
                 None => unreachable!(),
             },
@@ -473,40 +472,3 @@ async fn convert_streams<P: Into<PathBuf>>(
     }
     Ok(())
 }
-
-/*
-#[cfg(test)]
-mod tests {
-    use crate::client::RustyTube;
-
-    use super::*;
-    use indicatif::{ProgressDrawTarget, ProgressStyle};
-    use reqwest::ClientBuilder;
-
-    // #[test_log::test(tokio::test)]
-    #[tokio::test]
-    async fn t_download_video() {
-        let http = ClientBuilder::new()
-            .user_agent(
-                "Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0",
-            )
-            .gzip(true)
-            .brotli(true)
-            .build()
-            .expect("unable to build the HTTP client");
-
-        // Indicatif setup
-        let pb = ProgressBar::new(0);
-
-        let rt = RustyTube::new();
-        let player_data = rt
-            .get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop)
-            .await
-            .unwrap();
-
-        // download_video(&player_data, "tmp", "INVU", Some(1080), "ffmpeg", http, pb)
-        //     .await
-        //     .unwrap();
-    }
-}
-*/

downloader/src/util.rs

@@ -0,0 +1,42 @@
+use std::{borrow::Cow, collections::BTreeMap};
+
+use reqwest::Url;
+
+/// Error from the video downloader
+#[derive(thiserror::Error, Debug)]
+#[non_exhaustive]
+pub enum DownloadError {
+    /// Error from the HTTP client
+    #[error("http error: {0}")]
+    Http(#[from] reqwest::Error),
+    /// File IO error
+    #[error(transparent)]
+    Io(#[from] std::io::Error),
+    #[error("FFmpeg error: {0}")]
+    Ffmpeg(Cow<'static, str>),
+    #[error("Progressive download error: {0}")]
+    Progressive(Cow<'static, str>),
+    #[error("input error: {0}")]
+    Input(Cow<'static, str>),
+    #[error("error: {0}")]
+    Other(Cow<'static, str>),
+}
+
+/// Split an URL into its base string and parameter map
+///
+/// Example:
+///
+/// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}`
+pub fn url_to_params(url: &str) -> Result<(Url, BTreeMap<String, String>), DownloadError> {
+    let mut parsed_url = Url::parse(url).map_err(|e| {
+        DownloadError::Other(format!("could not parse url `{}` err: {}", url, e).into())
+    })?;
+    let url_params: BTreeMap<String, String> = parsed_url
+        .query_pairs()
+        .map(|(k, v)| (k.to_string(), v.to_string()))
+        .collect();
+
+    parsed_url.set_query(None);
+
+    Ok((parsed_url, url_params))
+}

src/error.rs

@@ -10,9 +10,6 @@ pub enum Error {
     /// Error from the deobfuscater
     #[error("deobfuscator error: {0}")]
     Deobfuscation(#[from] DeobfError),
-    /// Error from the video downloader
-    #[error("download error: {0}")]
-    Download(#[from] DownloadError),
     /// File IO error
     #[error(transparent)]
     Io(#[from] std::io::Error),
@@ -45,26 +42,6 @@ pub enum DeobfError {
     Other(&'static str),
 }
 
-/// Error from the video downloader
-#[derive(thiserror::Error, Debug)]
-#[non_exhaustive]
-pub enum DownloadError {
-    /// Error from the HTTP client
-    #[error("http error: {0}")]
-    Http(#[from] reqwest::Error),
-    /// File IO error
-    #[error(transparent)]
-    Io(#[from] std::io::Error),
-    #[error("FFmpeg error: {0}")]
-    Ffmpeg(Cow<'static, str>),
-    #[error("Progressive download error: {0}")]
-    Progressive(Cow<'static, str>),
-    #[error("input error: {0}")]
-    Input(Cow<'static, str>),
-    #[error("error: {0}")]
-    Other(Cow<'static, str>),
-}
-
 /// Error extracting content from YouTube
 #[derive(thiserror::Error, Debug)]
 #[non_exhaustive]

src/lib.rs

@@ -14,7 +14,6 @@ mod util;
 
 pub mod cache;
 pub mod client;
-pub mod download;
 pub mod error;
 pub mod model;
 pub mod param;