From aaa24bcc50184fa5c8cb2c479886f26058b603e9 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 10 Mar 2023 19:05:32 +0100 Subject: [PATCH] feat(cli): add getting youtube data --- cli/Cargo.toml | 3 + cli/src/main.rs | 385 +++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 315 insertions(+), 73 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0087f57..5fe99e2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -17,3 +17,6 @@ futures = "0.3.21" anyhow = "1.0" clap = { version = "4.0.29", features = ["derive"] } env_logger = "0.10.0" +serde = "1.0" +serde_json = "1.0.82" +serde_yaml = "0.9.19" diff --git a/cli/src/main.rs b/cli/src/main.rs index 578cbaa..6e57f8c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,45 +1,96 @@ use std::path::PathBuf; use anyhow::{Context, Result}; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use futures::stream::{self, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{Client, ClientBuilder}; -use rustypipe::{client::RustyPipe, param::StreamFilter}; +use rustypipe::{ + client::RustyPipe, + model::{UrlTarget, VideoId}, + param::StreamFilter, +}; +use serde::Serialize; #[derive(Parser)] #[clap(author, version, about, long_about = None)] struct Cli { #[clap(subcommand)] command: Commands, - #[clap(short, value_parser, default_value = ".", global = true)] - output: PathBuf, - #[clap(long, value_parser, global = true)] - resolution: Option, - #[clap(short, long, value_parser, default_value = "8", global = true)] - parallel: usize, } #[derive(Subcommand)] enum Commands { - /// Download a playlist - Playlist { - /// Playlist ID - #[clap(value_parser)] + /// Download a video, playlist, album or channel + #[clap(alias = "dl")] + Download { + /// ID or URL id: String, + /// Output path + #[clap(short, default_value = ".")] + output: PathBuf, + /// Video resolution (e.g. 720, 1080). Set to 0 for audio-only. + #[clap(short, long)] + resolution: Option, + /// Number of videos downloaded in parallel + #[clap(short, long, default_value_t = 8)] + parallel: usize, + /// Limit the number of videos to download + #[clap(long, default_value_t = 1000)] + limit: usize, }, - /// Download a video - Video { - /// Video ID - #[clap(value_parser)] + /// Extract video, playlist, album or channel data + Get { + /// ID or URL id: String, + /// Output format + #[clap(long, value_parser, default_value = "json")] + format: Format, + /// Pretty-print output + #[clap(long)] + pretty: bool, + /// Limit the number of items to fetch + #[clap(long, default_value_t = 100)] + limit: usize, + /// Channel tab + #[clap(long, default_value = "videos")] + tab: ChannelTab, + /// Use YouTube Music + #[clap(long)] + music: bool, + /// Get comments + #[clap(long)] + comments: Option, + /// Get lyrics + #[clap(long)] + lyrics: bool, }, } +#[derive(Copy, Clone, ValueEnum)] +enum Format { + Json, + Yaml, +} + +#[derive(Copy, Clone, ValueEnum)] +enum ChannelTab { + Videos, + Shorts, + Live, + Info, +} + +#[derive(Copy, Clone, ValueEnum)] +enum CommentsOrder { + Top, + Latest, +} + #[allow(clippy::too_many_arguments)] async fn download_single_video( - video_id: String, - video_title: String, + video_id: &str, + video_title: &str, output_dir: &str, output_fname: Option, resolution: Option, @@ -57,7 +108,7 @@ async fn download_single_video( let res = async { let player_data = rp .query() - .player(video_id.as_str()) + .player(video_id) .await .context(format!("Failed to fetch player data for video {video_id}"))?; @@ -94,7 +145,22 @@ async fn download_single_video( res } +fn print_data(data: &T, format: Format, pretty: bool) { + let stdout = std::io::stdout().lock(); + match format { + Format::Json => { + if pretty { + serde_json::to_writer_pretty(stdout, data).unwrap() + } else { + serde_json::to_writer(stdout, data).unwrap() + } + } + Format::Yaml => serde_yaml::to_writer(stdout, data).unwrap(), + }; +} + async fn download_video( + rp: &RustyPipe, id: &str, output_dir: &str, output_fname: Option, @@ -107,19 +173,17 @@ async fn download_video( .build() .expect("unable to build the HTTP client"); - let rp = RustyPipe::default(); - // Indicatif setup let multi = MultiProgress::new(); download_single_video( - id.to_owned(), - id.to_owned(), + id, + id, output_dir, output_fname, resolution, "ffmpeg", - &rp, + rp, http, multi, None, @@ -128,8 +192,9 @@ async fn download_video( .unwrap_or_else(|e| println!("ERROR: {e:?}")); } -async fn download_playlist( - id: &str, +async fn download_videos( + rp: &RustyPipe, + videos: &[VideoId], output_dir: &str, output_fname: Option, resolution: Option, @@ -142,18 +207,10 @@ async fn download_playlist( .build() .expect("unable to build the HTTP client"); - let rp = RustyPipe::default(); - let mut playlist = rp.query().playlist(id).await.unwrap(); - playlist - .videos - .extend_pages(&rp.query(), usize::MAX) - .await - .unwrap(); - // Indicatif setup let multi = MultiProgress::new(); let main = multi.add(ProgressBar::new( - playlist.videos.items.len().try_into().unwrap_or_default(), + videos.len().try_into().unwrap_or_default(), )); main.set_style( @@ -164,16 +221,16 @@ async fn download_playlist( ); main.tick(); - stream::iter(playlist.videos.items) + stream::iter(videos) .map(|video| { download_single_video( - video.id, - video.name, + &video.id, + &video.name, output_dir, output_fname.to_owned(), resolution, "ffmpeg", - &rp, + rp, http.clone(), multi.clone(), Some(main.clone()), @@ -197,43 +254,225 @@ async fn main() { let cli = Cli::parse(); - // Cases: Existing folder, non-existing file with existing parent folder, - // Error cases: non-existing parent folder, existing file - let output_path = std::fs::canonicalize(cli.output).unwrap(); - if output_path.is_file() { - println!("Output file already exists"); - return; - } - let (output_dir, output_fname) = if output_path.is_dir() { - (output_path.to_string_lossy().to_string(), None) - } else { - let output_dir_parent = output_path.parent().unwrap(); - if !output_dir_parent.is_dir() { - println!( - "Parent folder {} does not exist", - output_dir_parent.to_string_lossy() - ); - return; - } - - ( - output_dir_parent.to_string_lossy().to_string(), - Some( - output_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(), - ), - ) - }; + let rp = RustyPipe::new(); match cli.command { - Commands::Playlist { id } => { - download_playlist(&id, &output_dir, output_fname, cli.resolution, cli.parallel).await + Commands::Download { + id, + output, + resolution, + parallel, + limit, + } => { + // Cases: Existing folder, non-existing file with existing parent folder, + // Error cases: non-existing parent folder, existing file + let output_path = std::fs::canonicalize(output).unwrap(); + if output_path.is_file() { + println!("Output file already exists"); + return; + } + let (output_dir, output_fname) = if output_path.is_dir() { + (output_path.to_string_lossy().to_string(), None) + } else { + let output_dir_parent = output_path.parent().unwrap(); + if !output_dir_parent.is_dir() { + println!( + "Parent folder {} does not exist", + output_dir_parent.to_string_lossy() + ); + return; + } + + ( + output_dir_parent.to_string_lossy().to_string(), + Some( + output_path + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + ), + ) + }; + + let target = rp.query().resolve_string(&id, false).await.unwrap(); + match target { + UrlTarget::Video { id, .. } => { + download_video(&rp, &id, &output_dir, output_fname, resolution).await; + } + UrlTarget::Channel { id } => { + let mut channel = rp.query().channel_videos(id).await.unwrap(); + channel + .content + .extend_limit(&rp.query(), limit) + .await + .unwrap(); + let videos: Vec = channel + .content + .items + .into_iter() + .take(limit) + .map(VideoId::from) + .collect(); + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; + } + UrlTarget::Playlist { id } => { + let mut playlist = rp.query().playlist(id).await.unwrap(); + playlist + .videos + .extend_limit(&rp.query(), limit) + .await + .unwrap(); + let videos: Vec = playlist + .videos + .items + .into_iter() + .take(limit) + .map(VideoId::from) + .collect(); + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; + } + UrlTarget::Album { id } => { + let album = rp.query().music_album(id).await.unwrap(); + let videos: Vec = album + .tracks + .into_iter() + .take(limit) + .map(VideoId::from) + .collect(); + download_videos( + &rp, + &videos, + &output_dir, + output_fname, + resolution, + parallel, + ) + .await; + } + } } - Commands::Video { id } => { - download_video(&id, &output_dir, output_fname, cli.resolution).await + Commands::Get { + id, + format, + pretty, + limit, + tab, + music, + comments, + lyrics, + } => { + let target = rp.query().resolve_string(&id, false).await.unwrap(); + + match target { + UrlTarget::Video { id, .. } => { + if lyrics { + let details = rp.query().music_details(&id).await.unwrap(); + match details.lyrics_id { + Some(lyrics_id) => { + let lyrics = rp.query().music_lyrics(lyrics_id).await.unwrap(); + print_data(&lyrics, format, pretty); + } + None => eprintln!("no lyrics found"), + } + } else if music { + let details = rp.query().music_details(&id).await.unwrap(); + print_data(&details, format, pretty); + } else { + let mut details = rp.query().video_details(&id).await.unwrap(); + + match comments { + Some(CommentsOrder::Top) => { + details + .top_comments + .extend_limit(rp.query(), limit) + .await + .unwrap(); + } + Some(CommentsOrder::Latest) => { + details + .latest_comments + .extend_limit(rp.query(), limit) + .await + .unwrap(); + } + None => {} + } + + print_data(&details, format, pretty); + } + } + UrlTarget::Channel { id } => { + if music { + let artist = rp.query().music_artist(&id, true).await.unwrap(); + print_data(&artist, format, pretty); + } else { + match tab { + ChannelTab::Videos => { + let mut channel = rp.query().channel_videos(&id).await.unwrap(); + channel + .content + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&channel, format, pretty); + } + ChannelTab::Shorts => { + let mut channel = rp.query().channel_shorts(&id).await.unwrap(); + channel + .content + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&channel, format, pretty); + } + ChannelTab::Live => { + let mut channel = + rp.query().channel_livestreams(&id).await.unwrap(); + channel + .content + .extend_limit(rp.query(), limit) + .await + .unwrap(); + print_data(&channel, format, pretty); + } + ChannelTab::Info => { + let channel = rp.query().channel_info(&id).await.unwrap(); + print_data(&channel, format, pretty); + } + } + } + } + UrlTarget::Playlist { id } => { + if music { + let playlist = rp.query().music_playlist(&id).await.unwrap(); + print_data(&playlist, format, pretty); + } else { + let playlist = rp.query().playlist(&id).await.unwrap(); + print_data(&playlist, format, pretty); + } + } + UrlTarget::Album { id } => { + let album = rp.query().music_album(&id).await.unwrap(); + print_data(&album, format, pretty); + } + } } }; }