Sulkta-OSS/rustypipe
Archived
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add video cmd to cli

Browse source 
integrated stream filter
This commit is contained in:
ThetaDev 2022-08-27 00:28:50 +02:00
parent 30e79ebfbc
commit 5db85c05e8
3 changed files with 179 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

72
cli/src/main.rs
View file

@ -5,7 +5,10 @@ use clap::{Parser, Subcommand};
use futures::stream::{self, StreamExt}; use futures::stream::{self, StreamExt};
use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use reqwest::{Client, ClientBuilder}; use reqwest::{Client, ClientBuilder};
use rustypipe::client::{ClientType, RustyTube}; use rustypipe::{
client::{ClientType, RustyTube},
model::stream_filter::Filter,
};
#[derive(Parser)] #[derive(Parser)]
#[clap(author, version, about, long_about = None)] #[clap(author, version, about, long_about = None)]
@ -28,9 +31,15 @@ enum Commands {
#[clap(value_parser)] #[clap(value_parser)]
id: String, id: String,
}, },
/// Download a video
Video {
/// Video ID
#[clap(value_parser)]
id: String,
},
} }
async fn download_video( async fn download_single_video(
video_id: String, video_id: String,
video_title: String, video_title: String,
output_dir: &str, output_dir: &str,
@ -40,7 +49,7 @@ async fn download_video(
rt: &RustyTube, rt: &RustyTube,
http: Client, http: Client,
multi: MultiProgress, multi: MultiProgress,
main: ProgressBar, main: Option<ProgressBar>,
) -> Result<()> { ) -> Result<()> {
let pb = multi.add(ProgressBar::new(1)); let pb = multi.add(ProgressBar::new(1));
pb.set_style(ProgressStyle::default_bar() pb.set_style(ProgressStyle::default_bar()
@ -57,11 +66,21 @@ async fn download_video(
video_id video_id
))?; ))?;
let mut filter = Filter::default();
if let Some(res) = resolution {
if res == 0 {
filter.no_video();
} else {
filter.video_max_res(res);
}
}
rustypipe::download::download_video( rustypipe::download::download_video(
&player_data, &player_data,
output_dir, output_dir,
output_fname, output_fname,
resolution, None,
&filter,
ffmpeg, ffmpeg,
http, http,
pb, pb,
@ -74,10 +93,46 @@ async fn download_video(
} }
.await; .await;
main.inc(1); if let Some(main) = main {
main.inc(1);
}
res res
} }
async fn download_video(
id: &str,
output_dir: &str,
output_fname: Option<String>,
resolution: Option<u32>,
) {
let http = ClientBuilder::new()
.user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0")
.gzip(true)
.brotli(true)
.build()
.expect("unable to build the HTTP client");
let rt = RustyTube::new();
// Indicatif setup
let multi = MultiProgress::new();
download_single_video(
id.to_owned(),
id.to_owned(),
output_dir,
output_fname,
resolution,
"ffmpeg",
&rt,
http,
multi,
None,
)
.await
.unwrap_or_else(|e| println!("ERROR: {:?}", e));
}
async fn download_playlist( async fn download_playlist(
id: &str, id: &str,
output_dir: &str, output_dir: &str,
@ -111,7 +166,7 @@ async fn download_playlist(
stream::iter(playlist) stream::iter(playlist)
.map(|item| { .map(|item| {
download_video( download_single_video(
item.video_id.to_owned(), item.video_id.to_owned(),
item.title.to_owned(), item.title.to_owned(),
output_dir, output_dir,
@ -121,7 +176,7 @@ async fn download_playlist(
&rt, &rt,
http.clone(), http.clone(),
multi.clone(), multi.clone(),
main.clone(), Some(main.clone()),
) )
}) })
.buffer_unordered(parallel) .buffer_unordered(parallel)
@ -177,5 +232,8 @@ async fn main() {
Commands::Playlist { id } => { Commands::Playlist { id } => {
download_playlist(&id, &output_dir, output_fname, cli.resolution, cli.parallel).await download_playlist(&id, &output_dir, output_fname, cli.resolution, cli.parallel).await
} }
Commands::Video { id } => {
download_video(&id, &output_dir, output_fname, cli.resolution).await
}
}; };
} }

152
src/download.rs
View file

@ -15,7 +15,7 @@ use tokio::{
}; };
use crate::{ use crate::{
model::{AudioCodec, FileFormat, PlayerData, VideoCodec}, model::{stream_filter::Filter, AudioCodec, FileFormat, PlayerData, VideoCodec},
util, util,
}; };
@ -215,6 +215,7 @@ async fn download_chunks_by_param(
pb: ProgressBar, pb: ProgressBar,
) -> Result<()> { ) -> Result<()> {
let mut offset = offset; let mut offset = offset;
pb.inc_length(size);
loop { loop {
let range = get_download_range(offset, Some(size)); let range = get_download_range(offset, Some(size));
@ -260,7 +261,8 @@ pub async fn download_video(
player_data: &PlayerData, player_data: &PlayerData,
output_dir: &str, output_dir: &str,
output_fname: Option<String>, output_fname: Option<String>,
resolution: Option<u32>, output_format: Option<String>,
filter: &Filter,
ffmpeg: &str, ffmpeg: &str,
http: Client, http: Client,
pb: ProgressBar, pb: ProgressBar,
@ -273,39 +275,28 @@ pub async fn download_video(
.unwrap_or_else(|| filenamify::filenamify(format!("{} [{}]", title, player_data.info.id))); .unwrap_or_else(|| filenamify::filenamify(format!("{} [{}]", title, player_data.info.id)));
// Select streams to download // Select streams to download
let video = match resolution { let (video, audio) = player_data.select_video_audio_stream(filter);
Some(r) => Some(some_or_bail!(
player_data
.video_only_streams
.iter()
.rev()
.find(|s| s.height <= r && !s.hdr)
.clone(),
Err(anyhow!("no video stream matching res"))
)),
None => None,
};
let audio = some_or_bail!( if video.is_none() && audio.is_none() {
player_data return Err(anyhow!("no stream found"));
.audio_streams }
.iter()
.rev() let format = output_format.unwrap_or(
.filter(|a| a.codec != AudioCodec::Unknown) match video {
.next(), Some(_) => "mp4",
Err(anyhow!("no audio stream")) None => match audio {
Some(audio) => match audio.codec {
AudioCodec::Unknown => return Err(anyhow!("unknown audio codec")),
AudioCodec::Mp4a => "m4a",
AudioCodec::Opus => "opus",
},
None => unreachable!(),
},
}
.to_owned(),
); );
let format = match video { let output_path = download_dir.join(&output_fname).with_extension(&format);
Some(_) => "mp4",
None => match audio.codec {
AudioCodec::Unknown => panic!(),
AudioCodec::Mp4a => "m4a",
AudioCodec::Opus => "opus",
},
};
let output_path = download_dir.join(&output_fname).with_extension(format);
if output_path.exists() { if output_path.exists() {
// If the downloaded video already exists, only error if the download path was // If the downloaded video already exists, only error if the download path was
// chosen explicitly. // chosen explicitly.
@ -320,41 +311,63 @@ pub async fn download_video(
} }
} }
let mut downloads: Vec<StreamDownload> = Vec::new(); match (video, audio) {
// Downloading combined video/audio stream (no conversion)
(Some(video), None) => {
pb.set_message(format!("Downloading {}", title));
download_single_file(
&video.url,
download_dir.join(output_fname).with_extension(&format),
http,
pb.clone(),
)
.await?;
}
// Downloading split video/audio streams (requires conversion with ffmpeg)
_ => {
let mut downloads: Vec<StreamDownload> = Vec::new();
video.map(|v| { video.map(|v| {
downloads.push(StreamDownload { downloads.push(StreamDownload {
file: download_dir.join(format!("{}.video{}", output_fname, v.format.extension())), file: download_dir.join(format!(
url: v.url.to_owned(), "{}.video{}",
video_codec: Some(v.codec), output_fname,
audio_codec: None, v.format.extension()
}); )),
}); url: v.url.to_owned(),
downloads.push(StreamDownload { video_codec: Some(v.codec),
file: download_dir.join(format!( audio_codec: None,
"{}.audio{}", });
output_fname, });
audio.format.extension() audio.map(|a| {
)), downloads.push(StreamDownload {
url: audio.url.to_owned(), file: download_dir.join(format!(
video_codec: None, "{}.audio{}",
audio_codec: Some(audio.codec), output_fname,
}); a.format.extension()
)),
url: a.url.to_owned(),
video_codec: None,
audio_codec: Some(a.codec),
})
});
pb.set_message(format!("Downloading {}", title)); pb.set_message(format!("Downloading {}", title));
download_streams(&downloads, http, pb.clone()).await?; download_streams(&downloads, http, pb.clone()).await?;
pb.set_message(format!("Converting {}", title)); pb.set_message(format!("Converting {}", title));
convert_streams(&downloads, output_path, ffmpeg).await?; convert_streams(&downloads, output_path, ffmpeg).await?;
// Delete original files // Delete original files
stream::iter(&downloads) stream::iter(&downloads)
.map(|d| fs::remove_file(d.file.to_owned())) .map(|d| fs::remove_file(d.file.to_owned()))
.buffer_unordered(downloads.len()) .buffer_unordered(downloads.len())
.collect::<Vec<_>>() .collect::<Vec<_>>()
.await .await
.into_iter() .into_iter()
.collect::<Result<_, _>>()?; .collect::<Result<_, _>>()?;
}
}
pb.finish_and_clear(); pb.finish_and_clear();
Ok(()) Ok(())
@ -378,10 +391,6 @@ async fn download_streams(
Ok(()) Ok(())
} }
// ffmpeg -i TAEYEON\ 태연\ \'INVU\'\ MV.video.mp4
// -i TAEYEON\ 태연\ \'INVU\'\ MV.audio.webm -i hypa_audio.webm
// -map 0:v -map 1:a -map 2:a -metadata:s:a:1 language=en
// -metadata:s:a:2 language=de -c copy multiaudio.mp4
async fn convert_streams<P: Into<PathBuf>>( async fn convert_streams<P: Into<PathBuf>>(
downloads: &Vec<StreamDownload>, downloads: &Vec<StreamDownload>,
output: P, output: P,
@ -391,7 +400,6 @@ async fn convert_streams<P: Into<PathBuf>>(
let mut args: Vec<OsString> = vec![]; let mut args: Vec<OsString> = vec![];
let mut mapping_args: Vec<OsString> = vec![]; let mut mapping_args: Vec<OsString> = vec![];
// let mut meta_args: Vec<OsString> = vec![];
downloads.iter().enumerate().for_each(|(i, d)| { downloads.iter().enumerate().for_each(|(i, d)| {
args.push("-i".into()); args.push("-i".into());
@ -403,8 +411,12 @@ async fn convert_streams<P: Into<PathBuf>>(
args.append(&mut mapping_args); args.append(&mut mapping_args);
args.push("-c".into()); // Combining multiple streams, keep codecs
args.push("copy".into()); if downloads.len() > 1 {
args.push("-c".into());
args.push("copy".into());
}
args.push(output_path.into()); args.push(output_path.into());
let res = Command::new(ffmpeg).args(args).output().await?; let res = Command::new(ffmpeg).args(args).output().await?;

53
src/model/stream_filter.rs
View file

@ -15,6 +15,7 @@ pub struct Filter {
video_formats: Option<HashSet<VideoFormat>>, video_formats: Option<HashSet<VideoFormat>>,
video_codecs: Option<HashSet<VideoCodec>>, video_codecs: Option<HashSet<VideoCodec>>,
video_hdr: bool, video_hdr: bool,
video_none: bool,
} }
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
@ -204,6 +205,12 @@ impl Filter {
} }
} }
/// Output no video stream (audio only)
pub fn no_video(&mut self) -> &mut Self {
self.video_none = true;
self
}
fn apply_audio(&self, stream: &AudioStream) -> FilterResult { fn apply_audio(&self, stream: &AudioStream) -> FilterResult {
self.apply_audio_max_bitrate(stream).join( self.apply_audio_max_bitrate(stream).join(
self.apply_audio_formats(stream).join( self.apply_audio_formats(stream).join(
@ -253,6 +260,10 @@ impl PlayerData {
streams: &'a [VideoStream], streams: &'a [VideoStream],
filter: &Filter, filter: &Filter,
) -> Option<&'a VideoStream> { ) -> Option<&'a VideoStream> {
if filter.video_none {
return None;
}
let mut fallback: Option<&VideoStream> = None; let mut fallback: Option<&VideoStream> = None;
streams streams
@ -286,26 +297,26 @@ impl PlayerData {
pub fn select_video_audio_stream( pub fn select_video_audio_stream(
&self, &self,
filter: &Filter, filter: &Filter,
) -> Option<(&VideoStream, Option<&AudioStream>)> { ) -> (Option<&VideoStream>, Option<&AudioStream>) {
let video_stream = self.select_video_stream(filter); let video_stream = self.select_video_stream(filter);
let video_only_stream = self.select_video_only_stream(filter); let video_only_stream = self.select_video_only_stream(filter);
match (video_stream, video_only_stream) { match (video_stream, video_only_stream) {
(None, None) => None, (None, None) => (None, self.select_audio_stream(filter)),
(None, Some(video_only_stream)) => { (None, Some(video_only_stream)) => {
Some((video_only_stream, self.select_audio_stream(filter))) (Some(video_only_stream), self.select_audio_stream(filter))
} }
(Some(video_stream), None) => Some((video_stream, None)), (Some(video_stream), None) => (Some(video_stream), None),
(Some(video_stream), Some(video_only_stream)) => match video_only_stream > video_stream (Some(video_stream), Some(video_only_stream)) => match video_only_stream > video_stream
{ {
true => Some(( true => (
video_only_stream, Some(video_only_stream),
Some(some_or_bail!( Some(some_or_bail!(
self.select_audio_stream(filter), self.select_audio_stream(filter),
Some((video_stream, None)) (Some(video_stream), None)
)), )),
)), ),
false => Some((video_stream, None)), false => (Some(video_stream), None),
}, },
} }
} }
@ -386,6 +397,11 @@ mod tests {
Some("https://rr5---sn-h0jelne7.googlevideo.com/videoplayback?c=WEB&clen=23544588&dur=313.834&ei=eckIY72IKcGZ8gOMt6CwDg&expire=1661541849&fexp=24001373%2C24007246&fvip=2&gir=yes&id=o-AOqXE9lVS424yszv6LN5V_gaevdHxenJl-tYNy3Drs6g&initcwndbps=1428750&ip=2003%3Ade%3Aaf05%3A2500%3A5dad%3A319b%3Aca30%3Ae212&itag=18&lmt=1647456546485912&lsig=AG3C_xAwRQIhAMioKyc-dqs-6uvAwLViCcCTXKHn9sIbo0cbSSBXGG4kAiBQNsRBAvQrbWdOjZIsQXYrfPEb1KDpE_AlSEGQZXB9uA%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=NH&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jelne7%2Csn-h0jeenl6&ms=au%2Crdu&mt=1661519833&mv=m&mvi=5&n=HWZNhARNT_nJgg&ns=pLFQxzhiCbZ9F2HJmDLveKoH&pl=37&ratebypass=yes&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIgeCEjusAq6p33rH0NHyTAbPIRaaEkjDE32AXBFzDvR-ICIQD0LI8hQVH8oCMWu6OuADzc1FSQhIqYs5RLkxBmObIdsw%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cratebypass%2Cdur%2Clmt&spc=lT-KhuPtxVzL5-QbZ7S9zNeOHsWTdms&txp=4530434&vprv=1"), Some("https://rr5---sn-h0jelne7.googlevideo.com/videoplayback?c=WEB&clen=23544588&dur=313.834&ei=eckIY72IKcGZ8gOMt6CwDg&expire=1661541849&fexp=24001373%2C24007246&fvip=2&gir=yes&id=o-AOqXE9lVS424yszv6LN5V_gaevdHxenJl-tYNy3Drs6g&initcwndbps=1428750&ip=2003%3Ade%3Aaf05%3A2500%3A5dad%3A319b%3Aca30%3Ae212&itag=18&lmt=1647456546485912&lsig=AG3C_xAwRQIhAMioKyc-dqs-6uvAwLViCcCTXKHn9sIbo0cbSSBXGG4kAiBQNsRBAvQrbWdOjZIsQXYrfPEb1KDpE_AlSEGQZXB9uA%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=NH&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jelne7%2Csn-h0jeenl6&ms=au%2Crdu&mt=1661519833&mv=m&mvi=5&n=HWZNhARNT_nJgg&ns=pLFQxzhiCbZ9F2HJmDLveKoH&pl=37&ratebypass=yes&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIgeCEjusAq6p33rH0NHyTAbPIRaaEkjDE32AXBFzDvR-ICIQD0LI8hQVH8oCMWu6OuADzc1FSQhIqYs5RLkxBmObIdsw%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cratebypass%2Cdur%2Clmt&spc=lT-KhuPtxVzL5-QbZ7S9zNeOHsWTdms&txp=4530434&vprv=1"),
None None
)] )]
#[case::novideo(
Filter::default().no_video().to_owned(),
None,
Some("https://rr5---sn-h0jelne7.googlevideo.com/videoplayback?c=WEB&clen=5199784&dur=313.801&ei=eckIY72IKcGZ8gOMt6CwDg&expire=1661541849&fexp=24001373%2C24007246&fvip=2&gir=yes&id=o-AOqXE9lVS424yszv6LN5V_gaevdHxenJl-tYNy3Drs6g&initcwndbps=1428750&ip=2003%3Ade%3Aaf05%3A2500%3A5dad%3A319b%3Aca30%3Ae212&itag=251&keepalive=yes&lmt=1647453650291076&lsig=AG3C_xAwRQIhAMioKyc-dqs-6uvAwLViCcCTXKHn9sIbo0cbSSBXGG4kAiBQNsRBAvQrbWdOjZIsQXYrfPEb1KDpE_AlSEGQZXB9uA%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=NH&mime=audio%2Fwebm&mm=31%2C29&mn=sn-h0jelne7%2Csn-h0jeenl6&ms=au%2Crdu&mt=1661519833&mv=m&mvi=5&n=Zd7nrOM1B2C6PA&ns=426LxLap5MonJD_YWdS4lSYH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIhALtI3j8ZChpNb0LcyDZ3yosbWnSpqaO0-jKAe_UM_RQyAiAMwrpdeNbJEnQn3q1eveaAcRcNIwy5iJ4fIjeBW_MUfg%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-KhuPtxVzL5-QbZ7S9zNeOHsWTdms&txp=4532434&vprv=1")
)]
#[case::noformat(Filter::default().audio_formats(hash_set!()).video_formats(hash_set!()).to_owned(), None, None)] #[case::noformat(Filter::default().audio_formats(hash_set!()).video_formats(hash_set!()).to_owned(), None, None)]
fn t_select_video_audio_stream( fn t_select_video_audio_stream(
#[case] filter: Filter, #[case] filter: Filter,
@ -393,21 +409,16 @@ mod tests {
#[case] expect_audio_url: Option<&str>, #[case] expect_audio_url: Option<&str>,
) { ) {
let player_data = PLAYER_HDR; let player_data = PLAYER_HDR;
let selection = player_data.select_video_audio_stream(&filter); let (video, audio) = player_data.select_video_audio_stream(&filter);
match expect_video_url { match expect_video_url {
Some(expect_video_url) => { Some(expect_url) => assert_eq!(video.unwrap().url, expect_url),
let selection = selection.unwrap(); None => assert_eq!(video, None),
assert_eq!(selection.0.url, expect_video_url); }
match expect_audio_url { match expect_audio_url {
Some(expect_audio_url) => { Some(expect_url) => assert_eq!(audio.unwrap().url, expect_url),
assert_eq!(selection.1.unwrap().url, expect_audio_url) None => assert_eq!(audio, None),
}
None => assert_eq!(selection.1, None),
}
}
None => assert_eq!(selection, None),
} }
} }
} }