From 97fb0578b5c4954a596d8dee0c4b6e1d773a9300 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 6 Aug 2024 14:04:03 +0200 Subject: [PATCH] feat: add audiotag+indicatif features to downloader --- .forgejo/workflows/ci.yaml | 2 +- .pre-commit-config.yaml | 2 +- cli/src/main.rs | 101 +++++++++++-------------------------- downloader/README.md | 42 +++++++++++++++ downloader/src/lib.rs | 96 +++++++++++++++++++++++++---------- 5 files changed, 144 insertions(+), 99 deletions(-) create mode 100644 downloader/README.md diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 4239b3a..11582b8 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: cache-on-failure: "true" - name: ๐Ÿ“Ž Clippy - run: cargo clippy --all --features=rss -- -D warnings + run: cargo clippy --all --tests --features=rss,indicatif,audiotag -- -D warnings - name: ๐Ÿงช Test run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index defbeb7..05a4482 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,4 +10,4 @@ repos: hooks: - id: cargo-fmt - id: cargo-clippy - args: ["--all", "--tests", "--features=rss", "--", "-D", "warnings"] + args: ["--all", "--tests", "--features=rss,indicatif,audiotag", "--", "-D", "warnings"] diff --git a/cli/src/main.rs b/cli/src/main.rs index 3a8d503..3fbd68c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,4 @@ +#![doc = include_str!("../README.md")] #![warn(clippy::todo, clippy::dbg_macro)] use std::{path::PathBuf, str::FromStr, time::Duration}; @@ -10,7 +11,9 @@ use rustypipe::{ model::{UrlTarget, YouTubeItem}, param::{search_filter, ChannelVideoTab, Country, Language, StreamFilter}, }; -use rustypipe_downloader::{DownloadError, DownloadQuery, DownloadVideo, DownloaderBuilder}; +use rustypipe_downloader::{ + DownloadError, DownloadQuery, DownloadVideo, Downloader, DownloaderBuilder, +}; use serde::Serialize; use tracing::level_filters::LevelFilter; use tracing_subscriber::{fmt::MakeWriter, EnvFilter}; @@ -310,29 +313,12 @@ fn print_data(data: &T, format: Format, pretty: bool) { } async fn download_video( - rp: &RustyPipe, + dl: &Downloader, id: &str, target: &DownloadTarget, - resolution: Option, player_type: Option, - multi: MultiProgress, ) { - let mut filter = StreamFilter::new(); - if let Some(res) = resolution { - if res == 0 { - filter = filter.no_video(); - } else { - filter = filter.video_max_res(res); - } - } - let dl = DownloaderBuilder::new() - .rustypipe(rp) - .stream_filter(filter) - .progress_bar(multi) - .audio_tag() - .crop_cover() - .build(); - let mut q = target.apply(dl.download_id(id)); + let mut q = target.apply(dl.id(id)); if let Some(player_type) = player_type { q = q.player_type(player_type.into()); } @@ -343,31 +329,13 @@ async fn download_video( } async fn download_videos( - rp: &RustyPipe, + dl: &Downloader, videos: Vec, target: &DownloadTarget, - resolution: Option, parallel: usize, player_type: Option, multi: MultiProgress, ) { - let mut filter = StreamFilter::new(); - if let Some(res) = resolution { - if res == 0 { - filter = filter.no_video(); - } else { - filter = filter.video_max_res(res); - } - } - let dl = DownloaderBuilder::new() - .rustypipe(rp) - .stream_filter(filter) - .progress_bar(multi.clone()) - .audio_tag() - .crop_cover() - .path_precheck() - .build(); - // Indicatif setup let main = multi.add(ProgressBar::new( videos.len().try_into().unwrap_or_default(), @@ -387,7 +355,7 @@ async fn download_videos( let main = main.clone(); let id = video.id().to_owned(); - let mut q = target.apply(dl.download_video(video)); + let mut q = target.apply(dl.video(video)); if let Some(player_type) = player_type { q = q.player_type(player_type.into()); } @@ -470,9 +438,27 @@ async fn main() { player_type, } => { let url_target = rp.query().resolve_string(&id, false).await.unwrap(); + + let mut filter = StreamFilter::new(); + if let Some(res) = resolution { + if res == 0 { + filter = filter.no_video(); + } else { + filter = filter.video_max_res(res); + } + } + let dl = DownloaderBuilder::new() + .rustypipe(&rp) + .stream_filter(filter) + .multi_progress(multi.clone()) + .audio_tag() + .crop_cover() + .path_precheck() + .build(); + match url_target { UrlTarget::Video { id, .. } => { - download_video(&rp, &id, &target, resolution, player_type, multi).await; + download_video(&dl, &id, &target, player_type).await; } UrlTarget::Channel { id } => { target.assert_dir(); @@ -489,16 +475,7 @@ async fn main() { .take(limit) .map(|v| DownloadVideo::from_entity(&v)) .collect(); - download_videos( - &rp, - videos, - &target, - resolution, - parallel, - player_type, - multi, - ) - .await; + download_videos(&dl, videos, &target, parallel, player_type, multi).await; } UrlTarget::Playlist { id } => { target.assert_dir(); @@ -531,16 +508,7 @@ async fn main() { .map(|v| DownloadVideo::from_entity(&v)) .collect() }; - download_videos( - &rp, - videos, - &target, - resolution, - parallel, - player_type, - multi, - ) - .await; + download_videos(&dl, videos, &target, parallel, player_type, multi).await; } UrlTarget::Album { id } => { target.assert_dir(); @@ -551,16 +519,7 @@ async fn main() { .take(limit) .map(|v| DownloadVideo::from_track(&v)) .collect(); - download_videos( - &rp, - videos, - &target, - resolution, - parallel, - player_type, - multi, - ) - .await; + download_videos(&dl, videos, &target, parallel, player_type, multi).await; } } } diff --git a/downloader/README.md b/downloader/README.md new file mode 100644 index 0000000..24a554a --- /dev/null +++ b/downloader/README.md @@ -0,0 +1,42 @@ +# RustyPipe downloader + +The downloader is a companion crate for RustyPipe that allows for easy and fast +downloading of video and audio files. + +## Features + +- Fast download of streams, bypassing YouTube's throttling +- Join video and audio streams using ffmpeg +- [Indicatif](https://crates.io/crates/indicatif) support to show download progress bars + (enable `indicatif` feature to use) +- Tag audio files with title, album, artist, date, description and album cover (enable + `audiotag` feature to use) +- Album covers are automatically cropped using smartcrop to ensure they are square + +## How to use + +For the downloader to work, you need to have ffmpeg installed on your system. If your +ffmpeg binary is located at a non-standard path, you can configure the location using +[`DownloaderBuilder::ffmpeg`]. + +At first you have to instantiate and configure the downloader using either +[`Downloader::new`] or the [`DownloaderBuilder`]. + +Then you can build a new download query with a video ID, stream filter and destination +path and finally download the video. + +```rust ignore +use rustypipe::param::StreamFilter; +use rustypipe_downloader::DownloaderBuilder; + +let dl = DownloaderBuilder::new() + .audio_tag() + .crop_cover() + .build(); + +let filter_audio = StreamFilter::new().no_video(); +dl.id("ZeerrnuLi5E").stream_filter(filter_audio).to_file("audio.opus").download().await; + +let filter_video = StreamFilter::new().video_max_res(720); +dl.id("ZeerrnuLi5E").stream_filter(filter_video).to_file("video.mp4").download().await; +``` diff --git a/downloader/src/lib.rs b/downloader/src/lib.rs index d9588cd..685f69d 100644 --- a/downloader/src/lib.rs +++ b/downloader/src/lib.rs @@ -1,15 +1,12 @@ +#![doc = include_str!("../README.md")] #![warn(missing_docs, clippy::todo, clippy::dbg_macro)] -//! # YouTube audio/video downloader - mod util; use std::{ borrow::Cow, cmp::Ordering, ffi::OsString, - io::Cursor, - num::NonZeroU32, ops::Range, path::{Path, PathBuf}, sync::Arc, @@ -24,13 +21,11 @@ use reqwest::{header, Client, StatusCode}; use rustypipe::{ client::{ClientType, RustyPipe}, model::{ - richtext::ToPlaintext, traits::{FileFormat, YtEntity}, - AudioCodec, TrackItem, VideoCodec, VideoDetails, VideoPlayer, VideoPlayerDetails, + AudioCodec, TrackItem, VideoCodec, VideoPlayer, }, param::StreamFilter, }; -use time::{Date, OffsetDateTime}; use tokio::{ fs::{self, File}, io::AsyncWriteExt, @@ -42,6 +37,10 @@ use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; #[cfg(feature = "audiotag")] use lofty::{config::WriteOptions, picture::Picture, prelude::*, tag::Tag}; +#[cfg(feature = "audiotag")] +use rustypipe::model::{richtext::ToPlaintext, VideoDetails, VideoPlayerDetails}; +#[cfg(feature = "audiotag")] +use time::{Date, OffsetDateTime}; pub use util::DownloadError; @@ -65,6 +64,8 @@ pub struct DownloaderBuilder { ffmpeg: String, #[cfg(feature = "indicatif")] multi: Option, + #[cfg(feature = "indicatif")] + progress_style: Option, filter: StreamFilter, video_format: DownloadVideoFormat, n_retries: u32, @@ -85,6 +86,9 @@ struct DownloaderInner { /// Global progress #[cfg(feature = "indicatif")] multi: Option, + /// Progress style + #[cfg(feature = "indicatif")] + progress_style: ProgressStyle, /// Default stream filter filter: StreamFilter, /// Default video format @@ -111,7 +115,7 @@ pub struct DownloadQuery { dest: DownloadDest, /// Progress bar #[cfg(feature = "indicatif")] - multi: Option, + progress: Option, /// Stream filter filter: Option, /// Target video format @@ -251,7 +255,7 @@ impl DownloadDest { }; if let Some(repl) = repl { acc += &repl; - acc += &ms[trimmed.len()..]; + acc += &ms[trimmed.len()..]; // preceeding whitespace } (acc, m.end()) }, @@ -277,6 +281,8 @@ impl Default for DownloaderBuilder { ffmpeg: "ffmpeg".to_owned(), #[cfg(feature = "indicatif")] multi: None, + #[cfg(feature = "indicatif")] + progress_style: None, filter: StreamFilter::new(), video_format: DownloadVideoFormat::Mp4, n_retries: 3, @@ -317,11 +323,19 @@ impl DownloaderBuilder { /// for all downloads #[cfg(feature = "indicatif")] #[must_use] - pub fn progress_bar(mut self, progress: MultiProgress) -> Self { + pub fn multi_progress(mut self, progress: MultiProgress) -> Self { self.multi = Some(progress); self } + /// Set the indicatif [`ProgressStyle`] for the progress bars displayed under `multi_progress` + #[cfg(feature = "indicatif")] + #[must_use] + pub fn progress_style(mut self, style: ProgressStyle) -> Self { + self.progress_style = Some(style); + self + } + /// Set the default [`StreamFilter`] for all downloads. /// /// The filter can be overridden for individual download queries. @@ -393,6 +407,12 @@ impl DownloaderBuilder { ffmpeg: self.ffmpeg, #[cfg(feature = "indicatif")] multi: self.multi, + #[cfg(feature = "indicatif")] + progress_style: self.progress_style.unwrap_or_else(|| { + ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})") + .unwrap() + .progress_chars("#>-") + }), filter: self.filter, video_format: self.video_format, n_retries: self.n_retries, @@ -431,7 +451,7 @@ impl Downloader { video, dest: DownloadDest::Default, #[cfg(feature = "indicatif")] - multi: None, + progress: None, filter: None, video_format: None, player_type: None, @@ -439,7 +459,8 @@ impl Downloader { } /// Download a video with the given ID - pub fn download_id>(&self, video_id: S) -> DownloadQuery { + #[must_use] + pub fn id>(&self, video_id: S) -> DownloadQuery { self.query(DownloadVideo { id: video_id.into(), ..Default::default() @@ -447,7 +468,8 @@ impl Downloader { } /// Download a video from a DownloadVideo object - pub fn download_video(&self, video: DownloadVideo) -> DownloadQuery { + #[must_use] + pub fn video(&self, video: DownloadVideo) -> DownloadQuery { self.query(video) } @@ -455,7 +477,8 @@ impl Downloader { /// /// Providing an entity has the advantage that the download path can be determined before the video /// is fetched, so already downloaded videos get skipped right away. - pub fn download_entity(&self, video: &impl YtEntity) -> DownloadQuery { + #[must_use] + pub fn entity(&self, video: &impl YtEntity) -> DownloadQuery { self.query(DownloadVideo::from_entity(video)) } @@ -465,7 +488,8 @@ impl Downloader { /// is fetched, so already downloaded videos get skipped right away. /// /// If an album track is downloaded, this method will also add the track number to the downloaded file - pub fn download_track(&self, track: &TrackItem) -> DownloadQuery { + #[must_use] + pub fn track(&self, track: &TrackItem) -> DownloadQuery { self.query(DownloadVideo::from_track(track)) } } @@ -495,6 +519,7 @@ impl DownloadQuery { /// /// Note that the file extension may be changed to fit the reuested video/audio format. /// Refer to the [`DownloadResult`] to get the actual path after downloading. + #[must_use] pub fn to_file>(mut self, file: P) -> Self { let file = file.into(); self.update_video_format(&file); @@ -504,15 +529,16 @@ impl DownloadQuery { /// Download to the given directory /// - /// The filename is created by this template: `{title} [{id}]`. + /// The filename is created by this template: `{track} {title} [{id}]`. /// /// You can use a custom filename template using [`DownloadQuery::to_template`] + #[must_use] pub fn to_dir>(mut self, dir: P) -> Self { self.dest = DownloadDest::Dir(dir.into()); self } - /// Download to the given filename template + /// Download to a path determined by a template /// /// Templates are paths that may contain variables for video metadata. /// @@ -521,9 +547,17 @@ impl DownloadQuery { /// - `{title}` Video title /// - `{channel}` Channel name /// - `{channel_id}` Channel ID + /// - `{album}` Album + /// - `{album_id}` Album ID + /// - `{track}` Track number + /// + /// Whitespace between template variables is automatically removed if a variable + /// contains no data (e.g. `{track} {name}` is equal to `{name}` if a video without + /// track number is downloaded). /// /// Note that the file extension may be changed to fit the reuested video/audio format. /// Refer to the [`DownloadResult`] to get the actual path after downloading. + #[must_use] pub fn to_template>(mut self, tmpl: P) -> Self { let tmpl = tmpl.into(); self.update_video_format(&tmpl); @@ -531,46 +565,52 @@ impl DownloadQuery { self } - /// Use a [`MultiProgress`] progress bar for all downloads + /// Show the progress of this download using a Indicatif [`ProgressBar`] #[cfg(feature = "indicatif")] - pub fn progress_bar(mut self, progress: MultiProgress) -> Self { - self.multi = Some(progress); + #[must_use] + pub fn progress_bar(mut self, progress: ProgressBar) -> Self { + self.progress = Some(progress); self } /// Set a [`StreamFilter`] for choosing a stream to be downloaded + #[must_use] pub fn stream_filter(mut self, filter: StreamFilter) -> Self { self.filter = Some(filter); self } /// Set the [`VideoFormat`] of downloaded videos + #[must_use] pub fn video_format(mut self, video_format: DownloadVideoFormat) -> Self { self.video_format = Some(video_format); self } /// Set the [`ClientType`] used to fetch the YT player + #[must_use] pub fn player_type(mut self, player_type: ClientType) -> Self { self.player_type = Some(player_type); self } /// Download the video + /// + /// If no download path is set, the video is downloaded to the current directory + /// with a filename created by this template: `{track} {title} [{id}]`. #[tracing::instrument(skip(self), fields(id = self.video.id))] pub async fn download(&self) -> Result { let mut last_err = None; // Progress bar #[cfg(feature = "indicatif")] - let pb = { - let multi = self.multi.clone().or_else(|| self.dl.i.multi.clone()); - multi.map(|m| { + let pb = match &self.progress { + Some(progress) => Some(progress.clone()), + None => self.dl.i.multi.clone().map(|m| { let pb = ProgressBar::new(1); - pb.set_style(ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap() - .progress_chars("#>-")); + pb.set_style(self.dl.i.progress_style.clone()); m.add(pb) - }) + }), }; for n in 0..=self.dl.i.n_retries { @@ -799,6 +839,8 @@ impl DownloadQuery { track: TrackItem, track_nr: Option, ) -> Result<()> { + use std::{io::Cursor, num::NonZeroU32}; + let mut tagged_file = lofty::read_from_path(file)?; let tag = match tagged_file.primary_tag_mut() { Some(primary_tag) => primary_tag, @@ -1306,9 +1348,11 @@ async fn convert_streams( Ok(()) } +#[cfg(feature = "audiotag")] const YMD_FORMAT: &[time::format_description::FormatItem] = time::macros::format_description!("[year]-[month]-[day]"); +#[cfg(feature = "audiotag")] fn extract_yt_release_date( description: &str, publish_date: Option,