Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add audiotag+indicatif features to downloader

Browse source
This commit is contained in:
ThetaDev 2024-08-06 14:04:03 +02:00
parent c6bd03fb70
commit 97fb0578b5
No known key found for this signature in database
GPG key ID: E319D3C5148D65B6
5 changed files with 144 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.forgejo/workflows/ci.yaml
View file

@ -17,7 +17,7 @@ jobs:
cache-on-failure: "true"
- name: 📎 Clippy
run: cargo clippy --all --features=rss -- -D warnings
run: cargo clippy --all --tests --features=rss,indicatif,audiotag -- -D warnings
- name: 🧪 Test
run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace

2
.pre-commit-config.yaml
View file

@ -10,4 +10,4 @@ repos:
hooks:
- id: cargo-fmt
- id: cargo-clippy
args: ["--all", "--tests", "--features=rss", "--", "-D", "warnings"]
args: ["--all", "--tests", "--features=rss,indicatif,audiotag", "--", "-D", "warnings"]

101
cli/src/main.rs
View file

@ -1,3 +1,4 @@
#![doc = include_str!("../README.md")]
#![warn(clippy::todo, clippy::dbg_macro)]
use std::{path::PathBuf, str::FromStr, time::Duration};
@ -10,7 +11,9 @@ use rustypipe::{
model::{UrlTarget, YouTubeItem},
param::{search_filter, ChannelVideoTab, Country, Language, StreamFilter},
};
use rustypipe_downloader::{DownloadError, DownloadQuery, DownloadVideo, DownloaderBuilder};
use rustypipe_downloader::{
DownloadError, DownloadQuery, DownloadVideo, Downloader, DownloaderBuilder,
};
use serde::Serialize;
use tracing::level_filters::LevelFilter;
use tracing_subscriber::{fmt::MakeWriter, EnvFilter};
@ -310,29 +313,12 @@ fn print_data<T: Serialize>(data: &T, format: Format, pretty: bool) {
}
async fn download_video(
rp: &RustyPipe,
dl: &Downloader,
id: &str,
target: &DownloadTarget,
resolution: Option<u32>,
player_type: Option<PlayerType>,
multi: MultiProgress,
) {
let mut filter = StreamFilter::new();
if let Some(res) = resolution {
if res == 0 {
filter = filter.no_video();
} else {
filter = filter.video_max_res(res);
}
}
let dl = DownloaderBuilder::new()
.rustypipe(rp)
.stream_filter(filter)
.progress_bar(multi)
.audio_tag()
.crop_cover()
.build();
let mut q = target.apply(dl.download_id(id));
let mut q = target.apply(dl.id(id));
if let Some(player_type) = player_type {
q = q.player_type(player_type.into());
}
@ -343,31 +329,13 @@ async fn download_video(
}
async fn download_videos(
rp: &RustyPipe,
dl: &Downloader,
videos: Vec<DownloadVideo>,
target: &DownloadTarget,
resolution: Option<u32>,
parallel: usize,
player_type: Option<PlayerType>,
multi: MultiProgress,
) {
let mut filter = StreamFilter::new();
if let Some(res) = resolution {
if res == 0 {
filter = filter.no_video();
} else {
filter = filter.video_max_res(res);
}
}
let dl = DownloaderBuilder::new()
.rustypipe(rp)
.stream_filter(filter)
.progress_bar(multi.clone())
.audio_tag()
.crop_cover()
.path_precheck()
.build();
// Indicatif setup
let main = multi.add(ProgressBar::new(
videos.len().try_into().unwrap_or_default(),
@ -387,7 +355,7 @@ async fn download_videos(
let main = main.clone();
let id = video.id().to_owned();
let mut q = target.apply(dl.download_video(video));
let mut q = target.apply(dl.video(video));
if let Some(player_type) = player_type {
q = q.player_type(player_type.into());
}
@ -470,9 +438,27 @@ async fn main() {
player_type,
} => {
let url_target = rp.query().resolve_string(&id, false).await.unwrap();
let mut filter = StreamFilter::new();
if let Some(res) = resolution {
if res == 0 {
filter = filter.no_video();
} else {
filter = filter.video_max_res(res);
}
}
let dl = DownloaderBuilder::new()
.rustypipe(&rp)
.stream_filter(filter)
.multi_progress(multi.clone())
.audio_tag()
.crop_cover()
.path_precheck()
.build();
match url_target {
UrlTarget::Video { id, .. } => {
download_video(&rp, &id, &target, resolution, player_type, multi).await;
download_video(&dl, &id, &target, player_type).await;
}
UrlTarget::Channel { id } => {
target.assert_dir();
@ -489,16 +475,7 @@ async fn main() {
.take(limit)
.map(|v| DownloadVideo::from_entity(&v))
.collect();
download_videos(
&rp,
videos,
&target,
resolution,
parallel,
player_type,
multi,
)
.await;
download_videos(&dl, videos, &target, parallel, player_type, multi).await;
}
UrlTarget::Playlist { id } => {
target.assert_dir();
@ -531,16 +508,7 @@ async fn main() {
.map(|v| DownloadVideo::from_entity(&v))
.collect()
};
download_videos(
&rp,
videos,
&target,
resolution,
parallel,
player_type,
multi,
)
.await;
download_videos(&dl, videos, &target, parallel, player_type, multi).await;
}
UrlTarget::Album { id } => {
target.assert_dir();
@ -551,16 +519,7 @@ async fn main() {
.take(limit)
.map(|v| DownloadVideo::from_track(&v))
.collect();
download_videos(
&rp,
videos,
&target,
resolution,
parallel,
player_type,
multi,
)
.await;
download_videos(&dl, videos, &target, parallel, player_type, multi).await;
}
}
}

42
downloader/README.md Normal file
View file

@ -0,0 +1,42 @@
# RustyPipe downloader
The downloader is a companion crate for RustyPipe that allows for easy and fast
downloading of video and audio files.
## Features
- Fast download of streams, bypassing YouTube's throttling
- Join video and audio streams using ffmpeg
- [Indicatif](https://crates.io/crates/indicatif) support to show download progress bars
(enable `indicatif` feature to use)
- Tag audio files with title, album, artist, date, description and album cover (enable
`audiotag` feature to use)
- Album covers are automatically cropped using smartcrop to ensure they are square
## How to use
For the downloader to work, you need to have ffmpeg installed on your system. If your
ffmpeg binary is located at a non-standard path, you can configure the location using
[`DownloaderBuilder::ffmpeg`].
At first you have to instantiate and configure the downloader using either
[`Downloader::new`] or the [`DownloaderBuilder`].
Then you can build a new download query with a video ID, stream filter and destination
path and finally download the video.
```rust ignore
use rustypipe::param::StreamFilter;
use rustypipe_downloader::DownloaderBuilder;
let dl = DownloaderBuilder::new()
.audio_tag()
.crop_cover()
.build();
let filter_audio = StreamFilter::new().no_video();
dl.id("ZeerrnuLi5E").stream_filter(filter_audio).to_file("audio.opus").download().await;
let filter_video = StreamFilter::new().video_max_res(720);
dl.id("ZeerrnuLi5E").stream_filter(filter_video).to_file("video.mp4").download().await;
```

96
downloader/src/lib.rs
View file

@ -1,15 +1,12 @@
#![doc = include_str!("../README.md")]
#![warn(missing_docs, clippy::todo, clippy::dbg_macro)]
//! # YouTube audio/video downloader
mod util;
use std::{
borrow::Cow,
cmp::Ordering,
ffi::OsString,
io::Cursor,
num::NonZeroU32,
ops::Range,
path::{Path, PathBuf},
sync::Arc,
@ -24,13 +21,11 @@ use reqwest::{header, Client, StatusCode};
use rustypipe::{
client::{ClientType, RustyPipe},
model::{
richtext::ToPlaintext,
traits::{FileFormat, YtEntity},
AudioCodec, TrackItem, VideoCodec, VideoDetails, VideoPlayer, VideoPlayerDetails,
AudioCodec, TrackItem, VideoCodec, VideoPlayer,
},
param::StreamFilter,
};
use time::{Date, OffsetDateTime};
use tokio::{
fs::{self, File},
io::AsyncWriteExt,
@ -42,6 +37,10 @@ use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
#[cfg(feature = "audiotag")]
use lofty::{config::WriteOptions, picture::Picture, prelude::*, tag::Tag};
#[cfg(feature = "audiotag")]
use rustypipe::model::{richtext::ToPlaintext, VideoDetails, VideoPlayerDetails};
#[cfg(feature = "audiotag")]
use time::{Date, OffsetDateTime};
pub use util::DownloadError;
@ -65,6 +64,8 @@ pub struct DownloaderBuilder {
ffmpeg: String,
#[cfg(feature = "indicatif")]
multi: Option<MultiProgress>,
#[cfg(feature = "indicatif")]
progress_style: Option<ProgressStyle>,
filter: StreamFilter,
video_format: DownloadVideoFormat,
n_retries: u32,
@ -85,6 +86,9 @@ struct DownloaderInner {
/// Global progress
#[cfg(feature = "indicatif")]
multi: Option<MultiProgress>,
/// Progress style
#[cfg(feature = "indicatif")]
progress_style: ProgressStyle,
/// Default stream filter
filter: StreamFilter,
/// Default video format
@ -111,7 +115,7 @@ pub struct DownloadQuery {
dest: DownloadDest,
/// Progress bar
#[cfg(feature = "indicatif")]
multi: Option<MultiProgress>,
progress: Option<ProgressBar>,
/// Stream filter
filter: Option<StreamFilter>,
/// Target video format
@ -251,7 +255,7 @@ impl DownloadDest {
};
if let Some(repl) = repl {
acc += &repl;
acc += &ms[trimmed.len()..];
acc += &ms[trimmed.len()..]; // preceeding whitespace
}
(acc, m.end())
},
@ -277,6 +281,8 @@ impl Default for DownloaderBuilder {
ffmpeg: "ffmpeg".to_owned(),
#[cfg(feature = "indicatif")]
multi: None,
#[cfg(feature = "indicatif")]
progress_style: None,
filter: StreamFilter::new(),
video_format: DownloadVideoFormat::Mp4,
n_retries: 3,
@ -317,11 +323,19 @@ impl DownloaderBuilder {
/// for all downloads
#[cfg(feature = "indicatif")]
#[must_use]
pub fn progress_bar(mut self, progress: MultiProgress) -> Self {
pub fn multi_progress(mut self, progress: MultiProgress) -> Self {
self.multi = Some(progress);
self
}
/// Set the indicatif [`ProgressStyle`] for the progress bars displayed under `multi_progress`
#[cfg(feature = "indicatif")]
#[must_use]
pub fn progress_style(mut self, style: ProgressStyle) -> Self {
self.progress_style = Some(style);
self
}
/// Set the default [`StreamFilter`] for all downloads.
///
/// The filter can be overridden for individual download queries.
@ -393,6 +407,12 @@ impl DownloaderBuilder {
ffmpeg: self.ffmpeg,
#[cfg(feature = "indicatif")]
multi: self.multi,
#[cfg(feature = "indicatif")]
progress_style: self.progress_style.unwrap_or_else(|| {
ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})")
.unwrap()
.progress_chars("#>-")
}),
filter: self.filter,
video_format: self.video_format,
n_retries: self.n_retries,
@ -431,7 +451,7 @@ impl Downloader {
video,
dest: DownloadDest::Default,
#[cfg(feature = "indicatif")]
multi: None,
progress: None,
filter: None,
video_format: None,
player_type: None,
@ -439,7 +459,8 @@ impl Downloader {
}
/// Download a video with the given ID
pub fn download_id<S: Into<String>>(&self, video_id: S) -> DownloadQuery {
#[must_use]
pub fn id<S: Into<String>>(&self, video_id: S) -> DownloadQuery {
self.query(DownloadVideo {
id: video_id.into(),
..Default::default()
@ -447,7 +468,8 @@ impl Downloader {
}
/// Download a video from a DownloadVideo object
pub fn download_video(&self, video: DownloadVideo) -> DownloadQuery {
#[must_use]
pub fn video(&self, video: DownloadVideo) -> DownloadQuery {
self.query(video)
}
@ -455,7 +477,8 @@ impl Downloader {
///
/// Providing an entity has the advantage that the download path can be determined before the video
/// is fetched, so already downloaded videos get skipped right away.
pub fn download_entity(&self, video: &impl YtEntity) -> DownloadQuery {
#[must_use]
pub fn entity(&self, video: &impl YtEntity) -> DownloadQuery {
self.query(DownloadVideo::from_entity(video))
}
@ -465,7 +488,8 @@ impl Downloader {
/// is fetched, so already downloaded videos get skipped right away.
///
/// If an album track is downloaded, this method will also add the track number to the downloaded file
pub fn download_track(&self, track: &TrackItem) -> DownloadQuery {
#[must_use]
pub fn track(&self, track: &TrackItem) -> DownloadQuery {
self.query(DownloadVideo::from_track(track))
}
}
@ -495,6 +519,7 @@ impl DownloadQuery {
///
/// Note that the file extension may be changed to fit the reuested video/audio format.
/// Refer to the [`DownloadResult`] to get the actual path after downloading.
#[must_use]
pub fn to_file<P: Into<PathBuf>>(mut self, file: P) -> Self {
let file = file.into();
self.update_video_format(&file);
@ -504,15 +529,16 @@ impl DownloadQuery {
/// Download to the given directory
///
/// The filename is created by this template: `{title} [{id}]`.
/// The filename is created by this template: `{track} {title} [{id}]`.
///
/// You can use a custom filename template using [`DownloadQuery::to_template`]
#[must_use]
pub fn to_dir<P: Into<PathBuf>>(mut self, dir: P) -> Self {
self.dest = DownloadDest::Dir(dir.into());
self
}
/// Download to the given filename template
/// Download to a path determined by a template
///
/// Templates are paths that may contain variables for video metadata.
///
@ -521,9 +547,17 @@ impl DownloadQuery {
/// - `{title}` Video title
/// - `{channel}` Channel name
/// - `{channel_id}` Channel ID
/// - `{album}` Album
/// - `{album_id}` Album ID
/// - `{track}` Track number
///
/// Whitespace between template variables is automatically removed if a variable
/// contains no data (e.g. `{track} {name}` is equal to `{name}` if a video without
/// track number is downloaded).
///
/// Note that the file extension may be changed to fit the reuested video/audio format.
/// Refer to the [`DownloadResult`] to get the actual path after downloading.
#[must_use]
pub fn to_template<P: Into<PathBuf>>(mut self, tmpl: P) -> Self {
let tmpl = tmpl.into();
self.update_video_format(&tmpl);
@ -531,46 +565,52 @@ impl DownloadQuery {
self
}
/// Use a [`MultiProgress`] progress bar for all downloads
/// Show the progress of this download using a Indicatif [`ProgressBar`]
#[cfg(feature = "indicatif")]
pub fn progress_bar(mut self, progress: MultiProgress) -> Self {
self.multi = Some(progress);
#[must_use]
pub fn progress_bar(mut self, progress: ProgressBar) -> Self {
self.progress = Some(progress);
self
}
/// Set a [`StreamFilter`] for choosing a stream to be downloaded
#[must_use]
pub fn stream_filter(mut self, filter: StreamFilter) -> Self {
self.filter = Some(filter);
self
}
/// Set the [`VideoFormat`] of downloaded videos
#[must_use]
pub fn video_format(mut self, video_format: DownloadVideoFormat) -> Self {
self.video_format = Some(video_format);
self
}
/// Set the [`ClientType`] used to fetch the YT player
#[must_use]
pub fn player_type(mut self, player_type: ClientType) -> Self {
self.player_type = Some(player_type);
self
}
/// Download the video
///
/// If no download path is set, the video is downloaded to the current directory
/// with a filename created by this template: `{track} {title} [{id}]`.
#[tracing::instrument(skip(self), fields(id = self.video.id))]
pub async fn download(&self) -> Result<DownloadResult> {
let mut last_err = None;
// Progress bar
#[cfg(feature = "indicatif")]
let pb = {
let multi = self.multi.clone().or_else(|| self.dl.i.multi.clone());
multi.map(|m| {
let pb = match &self.progress {
Some(progress) => Some(progress.clone()),
None => self.dl.i.multi.clone().map(|m| {
let pb = ProgressBar::new(1);
pb.set_style(ProgressStyle::with_template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap()
.progress_chars("#>-"));
pb.set_style(self.dl.i.progress_style.clone());
m.add(pb)
})
}),
};
for n in 0..=self.dl.i.n_retries {
@ -799,6 +839,8 @@ impl DownloadQuery {
track: TrackItem,
track_nr: Option<u16>,
) -> Result<()> {
use std::{io::Cursor, num::NonZeroU32};
let mut tagged_file = lofty::read_from_path(file)?;
let tag = match tagged_file.primary_tag_mut() {
Some(primary_tag) => primary_tag,
@ -1306,9 +1348,11 @@ async fn convert_streams(
Ok(())
}
#[cfg(feature = "audiotag")]
const YMD_FORMAT: &[time::format_description::FormatItem] =
time::macros::format_description!("[year]-[month]-[day]");
#[cfg(feature = "audiotag")]
fn extract_yt_release_date(
description: &str,
publish_date: Option<OffsetDateTime>,