first successful download
This commit is contained in:
ThetaDev
parent
a6041a013b
commit
beb1177a11
16 changed files with 4076 additions and 121 deletions
212
src/download.rs
212
src/download.rs
|
|
@ -1,15 +1,17 @@
|
|||
use std::{cmp::Ordering, ops::Range, path::PathBuf};
|
||||
use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf};
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use fancy_regex::Regex;
|
||||
use futures::stream::StreamExt;
|
||||
use indicatif::ProgressBar;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use log::debug;
|
||||
use once_cell::sync::Lazy;
|
||||
use rand::Rng;
|
||||
use reqwest::{header, Client};
|
||||
use tokio::{fs, io::AsyncWriteExt, process::Command};
|
||||
|
||||
use crate::model::{AudioCodec, FileFormat, PlayerData, VideoCodec};
|
||||
|
||||
const CHUNK_SIZE_MIN: u64 = 9000000;
|
||||
const CHUNK_SIZE_MAX: u64 = 11000000;
|
||||
|
||||
|
|
@ -45,19 +47,27 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
|
|||
))
|
||||
}
|
||||
|
||||
async fn download_single_file(
|
||||
url: &str,
|
||||
output: &str,
|
||||
async fn download_single_file<S: Into<String>, P: Into<PathBuf>>(
|
||||
url: S,
|
||||
output: P,
|
||||
http: Client,
|
||||
pb: ProgressBar,
|
||||
) -> Result<()> {
|
||||
// Check if file is already downloaded
|
||||
let output_path = PathBuf::from(output);
|
||||
let output_path: PathBuf = output.into();
|
||||
let url: String = url.into();
|
||||
|
||||
if output_path.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let output_path_tmp = PathBuf::from(output.to_owned() + ".part");
|
||||
let output_path_tmp = output_path.with_extension(format!(
|
||||
"{}.part",
|
||||
output_path
|
||||
.extension()
|
||||
.unwrap_or_default()
|
||||
.to_string_lossy()
|
||||
));
|
||||
let mut offset: u64 = 0;
|
||||
let mut size: Option<u64> = None;
|
||||
|
||||
|
|
@ -66,7 +76,7 @@ async fn download_single_file(
|
|||
let file_size = output_path_tmp.metadata()?.len();
|
||||
|
||||
let res = http
|
||||
.head(url)
|
||||
.head(url.to_owned())
|
||||
.header(header::RANGE, "bytes=0-0")
|
||||
.send()
|
||||
.await?
|
||||
|
|
@ -110,12 +120,17 @@ async fn download_single_file(
|
|||
.open(output_path_tmp.to_owned())
|
||||
.await?;
|
||||
|
||||
pb.set_style(ProgressStyle::default_bar()
|
||||
.template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap()
|
||||
.progress_chars("#>-"));
|
||||
pb.set_message("Downloading");
|
||||
|
||||
loop {
|
||||
let range = get_download_range(offset, size);
|
||||
debug!("Fetching range {}-{}", range.start, range.end);
|
||||
|
||||
let res = http
|
||||
.get(url)
|
||||
.get(url.to_owned())
|
||||
.header(header::ORIGIN, "https://www.youtube.com")
|
||||
.header(header::REFERER, "https://www.youtube.com/")
|
||||
.header(
|
||||
|
|
@ -135,7 +150,7 @@ async fn download_single_file(
|
|||
|
||||
let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?;
|
||||
|
||||
offset = parsed_offset;
|
||||
offset = parsed_offset + 1;
|
||||
if size.is_none() {
|
||||
size = Some(parsed_size);
|
||||
pb.inc_length(parsed_size);
|
||||
|
|
@ -150,7 +165,7 @@ async fn download_single_file(
|
|||
file.write_all_buf(&mut chunk).await?;
|
||||
}
|
||||
|
||||
if offset >= size.unwrap() - 1 {
|
||||
if offset >= size.unwrap() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -159,25 +174,125 @@ async fn download_single_file(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
// ffmpeg -i video.webm -i audio.webm -c copy output.mp4
|
||||
async fn join_video_audio(
|
||||
video_file: &str,
|
||||
audio_file: &str,
|
||||
output_file: &str,
|
||||
struct StreamDownload {
|
||||
file: PathBuf,
|
||||
// track_name: String TODO: add for multiple audio languages,
|
||||
url: String,
|
||||
audio_codec: Option<AudioCodec>,
|
||||
video_codec: Option<VideoCodec>,
|
||||
}
|
||||
|
||||
async fn download_video(
|
||||
player_data: &PlayerData,
|
||||
output_dir: &str,
|
||||
resolution: Option<u32>,
|
||||
ffmpeg: &str,
|
||||
http: Client,
|
||||
pb: ProgressBar,
|
||||
) -> Result<()> {
|
||||
// Select streams to download
|
||||
let video = match resolution {
|
||||
Some(r) => Some(some_or_bail!(
|
||||
player_data
|
||||
.video_only_streams
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|s| s.height == r && !s.hdr)
|
||||
.clone(),
|
||||
Err(anyhow!("no video stream matching res"))
|
||||
)),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let audio = some_or_bail!(
|
||||
player_data.audio_streams.iter().rev().next(),
|
||||
Err(anyhow!("no audio stream"))
|
||||
);
|
||||
|
||||
let download_dir = PathBuf::from(output_dir);
|
||||
let title_fname = player_data.info.title.to_owned(); // TODO: slugify
|
||||
|
||||
let mut downloads: Vec<StreamDownload> = Vec::new();
|
||||
|
||||
video.map(|v| {
|
||||
println!("Video: {}", v.url);
|
||||
downloads.push(StreamDownload {
|
||||
file: download_dir.join(format!("{}.video{}", title_fname, v.format.extension())),
|
||||
url: v.url.to_owned(),
|
||||
video_codec: Some(v.codec),
|
||||
audio_codec: None,
|
||||
});
|
||||
});
|
||||
println!("Audio: {}", audio.url);
|
||||
downloads.push(StreamDownload {
|
||||
file: download_dir.join(format!("{}.audio{}", title_fname, audio.format.extension())),
|
||||
url: audio.url.to_owned(),
|
||||
video_codec: None,
|
||||
audio_codec: Some(audio.codec),
|
||||
});
|
||||
|
||||
download_streams(&downloads, http, pb).await?;
|
||||
|
||||
let output_file = download_dir.join(format!("{}.mp4", title_fname));
|
||||
convert_streams(&downloads, output_file, ffmpeg).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_streams(
|
||||
downloads: &Vec<StreamDownload>,
|
||||
http: Client,
|
||||
pb: ProgressBar,
|
||||
) -> Result<()> {
|
||||
let n = downloads.len();
|
||||
|
||||
stream::iter(downloads)
|
||||
.map(|d| {
|
||||
download_single_file(
|
||||
d.url.to_owned(),
|
||||
d.file.to_owned(),
|
||||
http.clone(),
|
||||
pb.clone(),
|
||||
)
|
||||
})
|
||||
.buffer_unordered(n)
|
||||
.collect::<Vec<_>>()
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ffmpeg -i TAEYEON\ 태연\ \'INVU\'\ MV.video.mp4
|
||||
// -i TAEYEON\ 태연\ \'INVU\'\ MV.audio.webm -i hypa_audio.webm
|
||||
// -map 0:v -map 1:a -map 2:a -metadata:s:a:1 language=en
|
||||
// -metadata:s:a:2 language=de -c copy multiaudio.mp4
|
||||
async fn convert_streams<P: Into<PathBuf>>(
|
||||
downloads: &Vec<StreamDownload>,
|
||||
output: P,
|
||||
ffmpeg: &str,
|
||||
) -> Result<()> {
|
||||
let res = Command::new(ffmpeg)
|
||||
.args([
|
||||
"-i",
|
||||
video_file,
|
||||
"-i",
|
||||
audio_file,
|
||||
"-c",
|
||||
"copy",
|
||||
output_file,
|
||||
])
|
||||
.output()
|
||||
.await?;
|
||||
let output: PathBuf = output.into();
|
||||
let mut args: Vec<OsString> = vec![];
|
||||
let mut mapping_args: Vec<OsString> = vec![];
|
||||
// let mut meta_args: Vec<OsString> = vec![];
|
||||
|
||||
downloads.iter().enumerate().for_each(|(i, d)| {
|
||||
args.push("-i".into());
|
||||
args.push(d.file.to_owned().into());
|
||||
|
||||
mapping_args.push("-map".into());
|
||||
mapping_args.push(i.to_string().into());
|
||||
});
|
||||
|
||||
args.append(&mut mapping_args);
|
||||
|
||||
args.push("-c".into());
|
||||
args.push("copy".into());
|
||||
args.push(output.into());
|
||||
|
||||
let res = Command::new(ffmpeg).args(args).output().await?;
|
||||
|
||||
if !res.status.success() {
|
||||
bail!(
|
||||
|
|
@ -190,16 +305,17 @@ async fn join_video_audio(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::client::RustyTube;
|
||||
|
||||
use super::*;
|
||||
use indicatif::ProgressStyle;
|
||||
use indicatif::{ProgressDrawTarget, ProgressStyle};
|
||||
use reqwest::ClientBuilder;
|
||||
|
||||
const TEST_URL_AUDIO: &str = "https://rr5---sn-h0jeenl6.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=Wd3oYqnIHLKYx_APgPCeoAM&expire=1659449785&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AH9zSQFJUzAo61SdF1m4PUcknacuL35Mm8TgOmD5lfwF&initcwndbps=1597500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRgIhAOPc6qa-C6x1GOFxx5hpiP_ZFFeCAdHSr43mq4PujcasAiEA8NHcpNsurS187Gjg1WseiaQ_kslkKWU4fylIVGr4p8Y%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jeenl6%2Csn-4g5ednsl&ms=au%2Conr&mt=1659427257&mv=m&mvi=5&n=cRL0RZUaCeszsQ&ns=1UbvTJx8sEFT4vlb0jQyd68H&pl=37&requiressl=yes&sig=AOq0QJ8wRQIgRY8UR_GHs7T2ZX-0g6vRzvQS5MqpAMOs3sBpPthEzMUCIQDkh7aZOGpgzy82ha2CN2yiYS9NVHBd5WGa1e3K8GYKKg%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khs9YQMiuc_CePC7R74ycrwd1hNk&txp=4532434&vprv=1";
|
||||
const TEST_URL_VIDEO: &str = "https://rr5---sn-h0jeenl6.googlevideo.com/videoplayback?aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C271%2C278%2C313%2C394%2C395%2C396%2C397%2C398%2C399%2C400%2C401&c=WEB&clen=66413039&dur=229.270&ei=Wd3oYqnIHLKYx_APgPCeoAM&expire=1659449785&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AH9zSQFJUzAo61SdF1m4PUcknacuL35Mm8TgOmD5lfwF&initcwndbps=1597500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=248&keepalive=yes&lmt=1655512874472691&lsig=AG3C_xAwRAIgbmq3hI3VDXrOvENhCotYujpiKaJODqLVq-Il8K9OIwwCIHk-H0SzI4tH1w3TzKnVSbpjghk3AByD9VD75Ywii1F_&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=video%2Fwebm&mm=31%2C26&mn=sn-h0jeenl6%2Csn-4g5ednsl&ms=au%2Conr&mt=1659427257&mv=m&mvi=5&n=cRL0RZUaCeszsQ&ns=1UbvTJx8sEFT4vlb0jQyd68H&pl=37&requiressl=yes&sig=AOq0QJ8wRgIhAOuxn8gnk3FFCPPpEoylYPcLyas52BvyT7DzSAsbmJMIAiEAzUAnieCK31ZVQydfExQ5FSrCGJR3AzcwqgpENBzunjA%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khs9YQMiuc_CePC7R74ycrwd1hNk&txp=4537434&vprv=1";
|
||||
const TEST_URL_AUDIO: &str = "https://rr2---sn-h0jelnes.googlevideo.com/videoplayback?c=WEB&clen=3548576&dur=217.281&ei=XLTsYqrjBZWI6dsPpN2piAM&expire=1659701436&fexp=24001373%2C24007246&fvip=3&gir=yes&id=o-ADzcOIYmmZUru2VQVa-K0lhP_Uwt-YB868WY1tQpxP29&initcwndbps=1550000&ip=2003%3Ade%3Aaf09%3A3800%3Adf03%3Aff5b%3A9fbd%3Aef0b&itag=251&keepalive=yes&lmt=1655066322398609&lsig=AG3C_xAwRQIhAPWzFISUntnQVCePCtbi3PwsrztgOM_ACh3OQX333boNAiBHcu5TJj8oQGmgz8sfm_I9jkbiCM1VOq_vW-wN0ARlMg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=0P&mime=audio%2Fwebm&mm=31%2C29&mn=sn-h0jelnes%2Csn-h0jeened&ms=au%2Crdu&mt=1659679486&mv=m&mvi=2&n=9-E5diT6ORysAQ&ns=z1W4YnCGd7nB7ajH1gDgfDkH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRgIhAKd-cnF7ZCwKCi2J4_4R032sNFzquZUsgr0EStdolqETAiEAgBd-yD8HhXKiqll9_Pn_z2aWGBi1rcvqpO-KOsgaTZQ%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khvvt1xML3EE5f7dUNGCF9edAhhQ&txp=5532434&vp";
|
||||
const TEST_URL_VIDEO: &str = "https://rr2---sn-h0jelnes.googlevideo.com/videoplayback?aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C271%2C278%2C313%2C394%2C395%2C396%2C397%2C398%2C399%2C400%2C401&c=WEB&clen=53812383&dur=217.258&ei=XLTsYqrjBZWI6dsPpN2piAM&expire=1659701436&fexp=24001373%2C24007246&fvip=3&gir=yes&id=o-ADzcOIYmmZUru2VQVa-K0lhP_Uwt-YB868WY1tQpxP29&initcwndbps=1550000&ip=2003%3Ade%3Aaf09%3A3800%3Adf03%3Aff5b%3A9fbd%3Aef0b&itag=399&keepalive=yes&lmt=1655077485544227&lsig=AG3C_xAwRAIgYASOFHKLHNDlad52_t29Vem3WMdSI4n2cDkW_GxxGB0CICb1D5TmmApvKZQP-tf7Mq4pgYyA9ihm7Bx152GjrrFf&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=0P&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jelnes%2Csn-h0jeened&ms=au%2Crdu&mt=1659679486&mv=m&mvi=2&n=9-E5diT6ORysAQ&ns=z1W4YnCGd7nB7ajH1gDgfDkH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIgHo0czKIjgbtGJS9yQHRMHZyZ8tzRhgbxBAl2N39Ms0ICIQCSTqPrsewj0qYDxjXnp6nIuRkYZU6WTiHPeaXVz1-eEw%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khvvt1xML3EE5f7dUNGCF9edAhhQ&txp=5532434&vprv=1";
|
||||
|
||||
// #[tokio::test]
|
||||
async fn test() {
|
||||
// download_file(TEST_URL_LARGE, ".tmp/test.webm").await;
|
||||
#[test_log::test(tokio::test)]
|
||||
async fn t_download_video() {
|
||||
let http = ClientBuilder::new()
|
||||
.user_agent(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0",
|
||||
|
|
@ -211,25 +327,15 @@ mod tests {
|
|||
|
||||
// Indicatif setup
|
||||
let pb = ProgressBar::new(0);
|
||||
pb.set_style(ProgressStyle::default_bar()
|
||||
.template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap()
|
||||
.progress_chars("#>-"));
|
||||
pb.set_message("Downloading");
|
||||
|
||||
let (r1, r2) = tokio::join!(
|
||||
download_single_file(TEST_URL_VIDEO, "tmp/test.webm", http.clone(), pb.clone()),
|
||||
download_single_file(TEST_URL_AUDIO, "tmp/test_audio.webm", http, pb)
|
||||
);
|
||||
r1.unwrap();
|
||||
r2.unwrap();
|
||||
let rt = RustyTube::new();
|
||||
let player_data = rt
|
||||
.get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
join_video_audio(
|
||||
"tmp/test.webm",
|
||||
"tmp/test_audio.webm",
|
||||
"tmp/test.mp4",
|
||||
"ffmpeg",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
download_video(&player_data, "tmp", Some(1080), "ffmpeg", http, pb)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Reference in a new issue