first successful download

This commit is contained in:
ThetaDev 2022-08-06 23:37:27 +02:00
parent a6041a013b
commit beb1177a11
16 changed files with 4076 additions and 121 deletions

View file

@ -15,8 +15,8 @@ url = "2.2.2"
log = "0.4.17"
reqwest = {version = "0.11.11", features = ["json", "gzip", "brotli", "stream"]}
tokio = {version = "1.20.0", features = ["macros", "fs", "process"]}
serde_json = "1.0.82"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.82"
serde_with = {version = "2.0.0", features = ["json"] }
rand = "0.8.5"
async-trait = "0.1.56"

File diff suppressed because one or more lines are too long

View file

@ -3,6 +3,7 @@ Video: ZeerrnuLi5E
4K HDR: LXb3EKWsInQ
8K: Zv11L-ZfrSg
Music: ihUZMeYFZHA
Multilanguage: tVWWp1PqDus
# Livestreams
Live: 64DYi_8ESh0

View file

@ -14,19 +14,8 @@ use reqwest::Method;
use serde::Serialize;
use url::Url;
use super::{
response,
ClientType, ContextYT, RustyTube, YTClient,
};
use crate::{
client::response::player,
deobfuscate::Deobfuscator,
model::{
AudioCodec, AudioStream, PlayerData, Subtitle, Thumbnail, VideoCodec, VideoInfo,
VideoStream,
},
util,
};
use super::{response, ClientType, ContextYT, RustyTube, YTClient};
use crate::{client::response::player, deobfuscate::Deobfuscator, model::*, util};
// REQUEST
@ -193,6 +182,8 @@ fn map_video_stream(
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> Option<VideoStream> {
let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None);
Some(VideoStream {
url: some_or_bail!(map_url(f, deobf, last_nsig), None),
itag: f.itag,
@ -208,7 +199,8 @@ fn map_video_stream(
hdr: f.color_info.clone().unwrap_or_default().primaries
== player::Primaries::ColorPrimariesBt2020,
mime: f.mime_type.to_owned(),
codec: get_video_codec(&f.mime_type),
format: some_or_bail!(get_video_format(mtype), None),
codec: get_video_codec(codecs),
})
}
@ -217,6 +209,8 @@ fn map_audio_stream(
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> Option<AudioStream> {
let (mtype, codecs) = some_or_bail!(parse_mime(&f.mime_type), None);
Some(AudioStream {
url: some_or_bail!(map_url(f, deobf, last_nsig), None),
itag: f.itag,
@ -226,25 +220,38 @@ fn map_audio_stream(
index_range: f.index_range.to_owned(),
init_range: f.init_range.to_owned(),
mime: f.mime_type.to_owned(),
codec: get_audio_codec(&f.mime_type),
format: some_or_bail!(get_audio_format(mtype), None),
codec: get_audio_codec(codecs),
})
}
fn codecs_from_mime(mime: &str) -> Vec<&str> {
fn parse_mime(mime: &str) -> Option<(&str, Vec<&str>)> {
static PATTERN: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"(\w+/\w+);\scodecs="([a-zA-Z-0-9.,\s]*)""#).unwrap());
let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), vec![]);
captures
.get(2)
.unwrap()
.as_str()
.split(", ")
.collect::<Vec<&str>>()
let captures = some_or_bail!(PATTERN.captures(&mime).ok().flatten(), None);
Some((
captures.get(1).unwrap().as_str(),
captures
.get(2)
.unwrap()
.as_str()
.split(", ")
.collect::<Vec<&str>>(),
))
}
fn get_video_codec(mime: &str) -> VideoCodec {
for codec in codecs_from_mime(mime) {
fn get_video_format(mtype: &str) -> Option<VideoFormat> {
match mtype {
"video/3gpp" => Some(VideoFormat::ThreeGp),
"video/mp4" => Some(VideoFormat::Mp4),
"video/webm" => Some(VideoFormat::Webm),
_ => None,
}
}
fn get_video_codec(codecs: Vec<&str>) -> VideoCodec {
for codec in codecs {
if codec.starts_with("avc1") {
return VideoCodec::Avc1;
} else if codec.starts_with("vp9") || codec.starts_with("vp09") {
@ -258,8 +265,16 @@ fn get_video_codec(mime: &str) -> VideoCodec {
VideoCodec::Unknown
}
fn get_audio_codec(mime: &str) -> AudioCodec {
for codec in codecs_from_mime(mime) {
fn get_audio_format(mtype: &str) -> Option<AudioFormat> {
match mtype {
"audio/mp4" => Some(AudioFormat::M4a),
"audio/webm" => Some(AudioFormat::Webm),
_ => None,
}
}
fn get_audio_codec(codecs: Vec<&str>) -> AudioCodec {
for codec in codecs {
if codec.starts_with("mp4a") {
return AudioCodec::Mp4a;
} else if codec.starts_with("opus") {
@ -534,12 +549,14 @@ mod tests {
assert_eq!(video.quality, "720p");
assert_eq!(video.hdr, false);
assert_eq!(video.mime, "video/webm; codecs=\"vp09.00.31.08\"");
assert_eq!(video.format, VideoFormat::Webm);
assert_eq!(video.codec, VideoCodec::Vp9);
assert_eq!(audio.bitrate, 130685);
assert_eq!(audio.average_bitrate, 129496);
assert_eq!(audio.size, 4193863);
assert_eq!(audio.mime, "audio/mp4; codecs=\"mp4a.40.2\"");
assert_eq!(audio.format, AudioFormat::M4a);
assert_eq!(audio.codec, AudioCodec::Mp4a);
} else {
let video = player_data
@ -562,12 +579,14 @@ mod tests {
assert_eq!(video.quality, "720p");
assert_eq!(video.hdr, false);
assert_eq!(video.mime, "video/mp4; codecs=\"av01.0.05M.08\"");
assert_eq!(video.format, VideoFormat::Mp4);
assert_eq!(video.codec, VideoCodec::Av01);
assert_eq!(audio.bitrate, 142718);
assert_eq!(audio.average_bitrate, 130708);
assert_eq!(audio.size, 4232344);
assert_eq!(audio.mime, "audio/webm; codecs=\"opus\"");
assert_eq!(audio.format, AudioFormat::Webm);
assert_eq!(audio.codec, AudioCodec::Opus);
}

View file

@ -13,12 +13,18 @@ struct QPlaylist {
browse_id: String,
}
#[derive(Clone, Debug)]
pub struct TmpEntry {
title: String,
video_id: String,
}
impl RustyTube {
pub async fn get_playlist(
&self,
playlist_id: &str,
client_type: ClientType,
) -> Result<response::Playlist> {
) -> Result<Vec<TmpEntry>> {
// let client = self.desktop_client.clone();
let client = self.get_ytclient(client_type);
let context = client.get_context(true).await;
@ -38,7 +44,31 @@ impl RustyTube {
let playlist_response = resp.json::<response::Playlist>().await?;
Ok(playlist_response)
Ok(map_playlist_tmp(playlist_response))
}
}
fn map_playlist_tmp(response: response::Playlist) -> Vec<TmpEntry> {
let content = &response
.contents
.two_column_browse_results_renderer
.contents[0]
.tab_renderer
.content
.section_list_renderer
.contents[0];
match &content.item_section_renderer {
Some(items) => items.contents[0]
.playlist_video_list_renderer
.contents
.iter()
.map(|it| TmpEntry {
title: it.playlist_video_renderer.title.to_owned(),
video_id: it.playlist_video_renderer.video_id.to_owned(),
})
.collect(),
None => todo!(),
}
}
@ -51,7 +81,7 @@ mod tests {
use super::*;
#[allow(dead_code)]
// #[test_log::test(tokio::test)]
#[test_log::test(tokio::test)]
async fn download_testfiles() {
let tf_dir = Path::new("testfiles/playlist");
let playlist_id = "RDCLAK5uy_mHW5bcduhjB-PkTePAe6EoRMj1xNT8gzY";
@ -106,7 +136,7 @@ mod tests {
let playlist = rt
.get_playlist(
"RDCLAK5uy_mHW5bcduhjB-PkTePAe6EoRMj1xNT8gzY",
ClientType::DesktopMusic,
ClientType::Desktop,
)
.await
.unwrap();

View file

@ -29,32 +29,32 @@ pub struct Thumbnail {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MusicItem {
thumbnail: MusicThumbnailRenderer,
playlist_item_data: PlaylistItemData,
pub thumbnail: MusicThumbnailRenderer,
pub playlist_item_data: PlaylistItemData,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
flex_columns: Vec<MusicColumn>,
pub flex_columns: Vec<MusicColumn>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
fixed_columns: Vec<MusicColumn>,
pub fixed_columns: Vec<MusicColumn>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MusicThumbnailRenderer {
music_thumbnail_renderer: MusicThumbnailRenderer2,
pub music_thumbnail_renderer: MusicThumbnailRenderer2,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MusicThumbnailRenderer2 {
thumbnail: Thumbnails,
pub thumbnail: Thumbnails,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PlaylistItemData {
video_id: String,
pub video_id: String,
}
#[derive(Clone, Debug, Deserialize)]
@ -63,12 +63,12 @@ pub struct MusicColumn {
rename = "musicResponsiveListItemFlexColumnRenderer",
alias = "musicResponsiveListItemFixedColumnRenderer"
)]
renderer: MusicColumnRenderer,
pub renderer: MusicColumnRenderer,
}
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
pub struct MusicColumnRenderer {
#[serde_as(as = "crate::serializer::text::TextLink")]
text: TextLink,
pub text: TextLink,
}

View file

@ -235,5 +235,5 @@ pub struct PlayerMicroformatRenderer {
pub category: String,
pub publish_date: NaiveDate,
// Only on YT Music
pub tags: Option<Vec<String>>
pub tags: Option<Vec<String>>,
}

View file

@ -4,7 +4,7 @@ use serde_with::{json::JsonString, DefaultOnError, VecSkipError};
use crate::serializer::text::TextLink;
use super::{Thumbnails, MusicItem};
use super::{MusicItem, Thumbnails};
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -42,19 +42,19 @@ pub struct ItemSection {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PlaylistVideoList {
pub playlist_video_list_renderer: ContentsRenderer<PlaylistVideoItem>
pub playlist_video_list_renderer: ContentsRenderer<PlaylistVideoItem>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PlaylistVideoItem {
playlist_video_renderer: PlaylistVideo,
pub playlist_video_renderer: PlaylistVideo,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PlaylistMusicItem {
music_responsive_list_item_renderer: MusicItem,
pub music_responsive_list_item_renderer: MusicItem,
}
#[serde_as]
@ -94,12 +94,12 @@ pub struct HeaderRenderer {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ContentRenderer<T> {
pub content: T
pub content: T,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ContentsRenderer<T> {
#[serde(alias = "tabs")]
pub contents: Vec<T>
pub contents: Vec<T>,
}

View file

@ -57,7 +57,7 @@ impl Deobfuscator {
impl From<DeobfData> for Deobfuscator {
fn from(data: DeobfData) -> Self {
Self {data}
Self { data }
}
}
@ -379,7 +379,9 @@ c[36](c[8],c[32]),c[20](c[25],c[10]),c[2](c[22],c[8]),c[32](c[20],c[16]),c[32](c
async fn t_update() {
let client = Client::new();
let cache = Cache::default();
let deobf = Deobfuscator::from_fetched_info(client, cache).await.unwrap();
let deobf = Deobfuscator::from_fetched_info(client, cache)
.await
.unwrap();
let deobf_sig = deobf.deobfuscate_sig("GOqGOqGOq0QJ8wRAIgaryQHfplJ9xJSKFywyaSMHuuwZYsoMTAvRvfm51qIGECIA5061zWeyfMPX9hEl_U6f9J0tr7GTJMKyPf5XNrJb5fb5i").unwrap();
println!("{}", deobf_sig);

View file

@ -1,15 +1,17 @@
use std::{cmp::Ordering, ops::Range, path::PathBuf};
use std::{cmp::Ordering, ffi::OsString, ops::Range, path::PathBuf};
use anyhow::{anyhow, bail, Result};
use fancy_regex::Regex;
use futures::stream::StreamExt;
use indicatif::ProgressBar;
use futures::stream::{self, StreamExt};
use indicatif::{ProgressBar, ProgressStyle};
use log::debug;
use once_cell::sync::Lazy;
use rand::Rng;
use reqwest::{header, Client};
use tokio::{fs, io::AsyncWriteExt, process::Command};
use crate::model::{AudioCodec, FileFormat, PlayerData, VideoCodec};
const CHUNK_SIZE_MIN: u64 = 9000000;
const CHUNK_SIZE_MAX: u64 = 11000000;
@ -45,19 +47,27 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> {
))
}
async fn download_single_file(
url: &str,
output: &str,
async fn download_single_file<S: Into<String>, P: Into<PathBuf>>(
url: S,
output: P,
http: Client,
pb: ProgressBar,
) -> Result<()> {
// Check if file is already downloaded
let output_path = PathBuf::from(output);
let output_path: PathBuf = output.into();
let url: String = url.into();
if output_path.exists() {
return Ok(());
}
let output_path_tmp = PathBuf::from(output.to_owned() + ".part");
let output_path_tmp = output_path.with_extension(format!(
"{}.part",
output_path
.extension()
.unwrap_or_default()
.to_string_lossy()
));
let mut offset: u64 = 0;
let mut size: Option<u64> = None;
@ -66,7 +76,7 @@ async fn download_single_file(
let file_size = output_path_tmp.metadata()?.len();
let res = http
.head(url)
.head(url.to_owned())
.header(header::RANGE, "bytes=0-0")
.send()
.await?
@ -110,12 +120,17 @@ async fn download_single_file(
.open(output_path_tmp.to_owned())
.await?;
pb.set_style(ProgressStyle::default_bar()
.template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap()
.progress_chars("#>-"));
pb.set_message("Downloading");
loop {
let range = get_download_range(offset, size);
debug!("Fetching range {}-{}", range.start, range.end);
let res = http
.get(url)
.get(url.to_owned())
.header(header::ORIGIN, "https://www.youtube.com")
.header(header::REFERER, "https://www.youtube.com/")
.header(
@ -135,7 +150,7 @@ async fn download_single_file(
let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?;
offset = parsed_offset;
offset = parsed_offset + 1;
if size.is_none() {
size = Some(parsed_size);
pb.inc_length(parsed_size);
@ -150,7 +165,7 @@ async fn download_single_file(
file.write_all_buf(&mut chunk).await?;
}
if offset >= size.unwrap() - 1 {
if offset >= size.unwrap() {
break;
}
}
@ -159,25 +174,125 @@ async fn download_single_file(
Ok(())
}
// ffmpeg -i video.webm -i audio.webm -c copy output.mp4
async fn join_video_audio(
video_file: &str,
audio_file: &str,
output_file: &str,
struct StreamDownload {
file: PathBuf,
// track_name: String TODO: add for multiple audio languages,
url: String,
audio_codec: Option<AudioCodec>,
video_codec: Option<VideoCodec>,
}
async fn download_video(
player_data: &PlayerData,
output_dir: &str,
resolution: Option<u32>,
ffmpeg: &str,
http: Client,
pb: ProgressBar,
) -> Result<()> {
// Select streams to download
let video = match resolution {
Some(r) => Some(some_or_bail!(
player_data
.video_only_streams
.iter()
.rev()
.find(|s| s.height == r && !s.hdr)
.clone(),
Err(anyhow!("no video stream matching res"))
)),
None => None,
};
let audio = some_or_bail!(
player_data.audio_streams.iter().rev().next(),
Err(anyhow!("no audio stream"))
);
let download_dir = PathBuf::from(output_dir);
let title_fname = player_data.info.title.to_owned(); // TODO: slugify
let mut downloads: Vec<StreamDownload> = Vec::new();
video.map(|v| {
println!("Video: {}", v.url);
downloads.push(StreamDownload {
file: download_dir.join(format!("{}.video{}", title_fname, v.format.extension())),
url: v.url.to_owned(),
video_codec: Some(v.codec),
audio_codec: None,
});
});
println!("Audio: {}", audio.url);
downloads.push(StreamDownload {
file: download_dir.join(format!("{}.audio{}", title_fname, audio.format.extension())),
url: audio.url.to_owned(),
video_codec: None,
audio_codec: Some(audio.codec),
});
download_streams(&downloads, http, pb).await?;
let output_file = download_dir.join(format!("{}.mp4", title_fname));
convert_streams(&downloads, output_file, ffmpeg).await?;
Ok(())
}
async fn download_streams(
downloads: &Vec<StreamDownload>,
http: Client,
pb: ProgressBar,
) -> Result<()> {
let n = downloads.len();
stream::iter(downloads)
.map(|d| {
download_single_file(
d.url.to_owned(),
d.file.to_owned(),
http.clone(),
pb.clone(),
)
})
.buffer_unordered(n)
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<Vec<_>>>()?;
Ok(())
}
// ffmpeg -i TAEYEON\ 태연\ \'INVU\'\ MV.video.mp4
// -i TAEYEON\ 태연\ \'INVU\'\ MV.audio.webm -i hypa_audio.webm
// -map 0:v -map 1:a -map 2:a -metadata:s:a:1 language=en
// -metadata:s:a:2 language=de -c copy multiaudio.mp4
async fn convert_streams<P: Into<PathBuf>>(
downloads: &Vec<StreamDownload>,
output: P,
ffmpeg: &str,
) -> Result<()> {
let res = Command::new(ffmpeg)
.args([
"-i",
video_file,
"-i",
audio_file,
"-c",
"copy",
output_file,
])
.output()
.await?;
let output: PathBuf = output.into();
let mut args: Vec<OsString> = vec![];
let mut mapping_args: Vec<OsString> = vec![];
// let mut meta_args: Vec<OsString> = vec![];
downloads.iter().enumerate().for_each(|(i, d)| {
args.push("-i".into());
args.push(d.file.to_owned().into());
mapping_args.push("-map".into());
mapping_args.push(i.to_string().into());
});
args.append(&mut mapping_args);
args.push("-c".into());
args.push("copy".into());
args.push(output.into());
let res = Command::new(ffmpeg).args(args).output().await?;
if !res.status.success() {
bail!(
@ -190,16 +305,17 @@ async fn join_video_audio(
#[cfg(test)]
mod tests {
use crate::client::RustyTube;
use super::*;
use indicatif::ProgressStyle;
use indicatif::{ProgressDrawTarget, ProgressStyle};
use reqwest::ClientBuilder;
const TEST_URL_AUDIO: &str = "https://rr5---sn-h0jeenl6.googlevideo.com/videoplayback?c=WEB&clen=3781277&dur=229.301&ei=Wd3oYqnIHLKYx_APgPCeoAM&expire=1659449785&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AH9zSQFJUzAo61SdF1m4PUcknacuL35Mm8TgOmD5lfwF&initcwndbps=1597500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=251&keepalive=yes&lmt=1655510291473933&lsig=AG3C_xAwRgIhAOPc6qa-C6x1GOFxx5hpiP_ZFFeCAdHSr43mq4PujcasAiEA8NHcpNsurS187Gjg1WseiaQ_kslkKWU4fylIVGr4p8Y%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=audio%2Fwebm&mm=31%2C26&mn=sn-h0jeenl6%2Csn-4g5ednsl&ms=au%2Conr&mt=1659427257&mv=m&mvi=5&n=cRL0RZUaCeszsQ&ns=1UbvTJx8sEFT4vlb0jQyd68H&pl=37&requiressl=yes&sig=AOq0QJ8wRQIgRY8UR_GHs7T2ZX-0g6vRzvQS5MqpAMOs3sBpPthEzMUCIQDkh7aZOGpgzy82ha2CN2yiYS9NVHBd5WGa1e3K8GYKKg%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khs9YQMiuc_CePC7R74ycrwd1hNk&txp=4532434&vprv=1";
const TEST_URL_VIDEO: &str = "https://rr5---sn-h0jeenl6.googlevideo.com/videoplayback?aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C271%2C278%2C313%2C394%2C395%2C396%2C397%2C398%2C399%2C400%2C401&c=WEB&clen=66413039&dur=229.270&ei=Wd3oYqnIHLKYx_APgPCeoAM&expire=1659449785&fexp=24001373%2C24007246&fvip=5&gir=yes&id=o-AH9zSQFJUzAo61SdF1m4PUcknacuL35Mm8TgOmD5lfwF&initcwndbps=1597500&ip=2003%3Ade%3Aaf0e%3A2f00%3Ade47%3A297%3Aa6db%3A774e&itag=248&keepalive=yes&lmt=1655512874472691&lsig=AG3C_xAwRAIgbmq3hI3VDXrOvENhCotYujpiKaJODqLVq-Il8K9OIwwCIHk-H0SzI4tH1w3TzKnVSbpjghk3AByD9VD75Ywii1F_&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=hH&mime=video%2Fwebm&mm=31%2C26&mn=sn-h0jeenl6%2Csn-4g5ednsl&ms=au%2Conr&mt=1659427257&mv=m&mvi=5&n=cRL0RZUaCeszsQ&ns=1UbvTJx8sEFT4vlb0jQyd68H&pl=37&requiressl=yes&sig=AOq0QJ8wRgIhAOuxn8gnk3FFCPPpEoylYPcLyas52BvyT7DzSAsbmJMIAiEAzUAnieCK31ZVQydfExQ5FSrCGJR3AzcwqgpENBzunjA%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khs9YQMiuc_CePC7R74ycrwd1hNk&txp=4537434&vprv=1";
const TEST_URL_AUDIO: &str = "https://rr2---sn-h0jelnes.googlevideo.com/videoplayback?c=WEB&clen=3548576&dur=217.281&ei=XLTsYqrjBZWI6dsPpN2piAM&expire=1659701436&fexp=24001373%2C24007246&fvip=3&gir=yes&id=o-ADzcOIYmmZUru2VQVa-K0lhP_Uwt-YB868WY1tQpxP29&initcwndbps=1550000&ip=2003%3Ade%3Aaf09%3A3800%3Adf03%3Aff5b%3A9fbd%3Aef0b&itag=251&keepalive=yes&lmt=1655066322398609&lsig=AG3C_xAwRQIhAPWzFISUntnQVCePCtbi3PwsrztgOM_ACh3OQX333boNAiBHcu5TJj8oQGmgz8sfm_I9jkbiCM1VOq_vW-wN0ARlMg%3D%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=0P&mime=audio%2Fwebm&mm=31%2C29&mn=sn-h0jelnes%2Csn-h0jeened&ms=au%2Crdu&mt=1659679486&mv=m&mvi=2&n=9-E5diT6ORysAQ&ns=z1W4YnCGd7nB7ajH1gDgfDkH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRgIhAKd-cnF7ZCwKCi2J4_4R032sNFzquZUsgr0EStdolqETAiEAgBd-yD8HhXKiqll9_Pn_z2aWGBi1rcvqpO-KOsgaTZQ%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Citag%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khvvt1xML3EE5f7dUNGCF9edAhhQ&txp=5532434&vp";
const TEST_URL_VIDEO: &str = "https://rr2---sn-h0jelnes.googlevideo.com/videoplayback?aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C271%2C278%2C313%2C394%2C395%2C396%2C397%2C398%2C399%2C400%2C401&c=WEB&clen=53812383&dur=217.258&ei=XLTsYqrjBZWI6dsPpN2piAM&expire=1659701436&fexp=24001373%2C24007246&fvip=3&gir=yes&id=o-ADzcOIYmmZUru2VQVa-K0lhP_Uwt-YB868WY1tQpxP29&initcwndbps=1550000&ip=2003%3Ade%3Aaf09%3A3800%3Adf03%3Aff5b%3A9fbd%3Aef0b&itag=399&keepalive=yes&lmt=1655077485544227&lsig=AG3C_xAwRAIgYASOFHKLHNDlad52_t29Vem3WMdSI4n2cDkW_GxxGB0CICb1D5TmmApvKZQP-tf7Mq4pgYyA9ihm7Bx152GjrrFf&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl%2Cinitcwndbps&mh=0P&mime=video%2Fmp4&mm=31%2C29&mn=sn-h0jelnes%2Csn-h0jeened&ms=au%2Crdu&mt=1659679486&mv=m&mvi=2&n=9-E5diT6ORysAQ&ns=z1W4YnCGd7nB7ajH1gDgfDkH&pl=37&rbqsm=fr&requiressl=yes&sig=AOq0QJ8wRQIgHo0czKIjgbtGJS9yQHRMHZyZ8tzRhgbxBAl2N39Ms0ICIQCSTqPrsewj0qYDxjXnp6nIuRkYZU6WTiHPeaXVz1-eEw%3D%3D&source=youtube&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&spc=lT-Khvvt1xML3EE5f7dUNGCF9edAhhQ&txp=5532434&vprv=1";
// #[tokio::test]
async fn test() {
// download_file(TEST_URL_LARGE, ".tmp/test.webm").await;
#[test_log::test(tokio::test)]
async fn t_download_video() {
let http = ClientBuilder::new()
.user_agent(
"Mozilla/5.0 (Windows NT 10.0; Win64; rv:107.0) Gecko/20100101 Firefox/107.0",
@ -211,25 +327,15 @@ mod tests {
// Indicatif setup
let pb = ProgressBar::new(0);
pb.set_style(ProgressStyle::default_bar()
.template("{msg}\n{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})").unwrap()
.progress_chars("#>-"));
pb.set_message("Downloading");
let (r1, r2) = tokio::join!(
download_single_file(TEST_URL_VIDEO, "tmp/test.webm", http.clone(), pb.clone()),
download_single_file(TEST_URL_AUDIO, "tmp/test_audio.webm", http, pb)
);
r1.unwrap();
r2.unwrap();
let rt = RustyTube::new();
let player_data = rt
.get_player("AbZH7XWDW_k", crate::client::ClientType::Desktop)
.await
.unwrap();
join_video_audio(
"tmp/test.webm",
"tmp/test_audio.webm",
"tmp/test.mp4",
"ffmpeg",
)
.await
.unwrap();
download_video(&player_data, "tmp", Some(1080), "ffmpeg", http, pb)
.await
.unwrap();
}
}

View file

@ -3,11 +3,11 @@
#[macro_use]
mod macros;
mod util;
mod serializer;
mod cache;
mod deobfuscate;
mod serializer;
mod util;
pub mod model;
pub mod client;
pub mod download;
pub mod model;

View file

@ -1,7 +1,11 @@
use std::ops::Range;
use chrono::{DateTime, Utc};
use serde::{Serialize, Deserialize};
use serde::{Deserialize, Serialize};
pub trait FileFormat {
fn extension(&self) -> &str;
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PlayerData {
@ -30,7 +34,6 @@ pub struct VideoInfo {
pub category: Option<String>,
pub is_live_content: bool,
pub is_family_safe: Option<bool>,
// pub like_count: Option<u32>,
// pub dislike_count: Option<u32>
}
@ -50,6 +53,7 @@ pub struct VideoStream {
pub quality: String,
pub hdr: bool,
pub mime: String,
pub format: VideoFormat,
pub codec: VideoCodec,
}
@ -63,10 +67,13 @@ pub struct AudioStream {
pub index_range: Option<Range<u32>>,
pub init_range: Option<Range<u32>>,
pub mime: String,
pub format: AudioFormat,
pub codec: AudioCodec,
}
#[derive(Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(
Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum VideoCodec {
@ -82,7 +89,9 @@ pub enum VideoCodec {
Av01,
}
#[derive(Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(
Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum AudioCodec {
@ -91,7 +100,45 @@ pub enum AudioCodec {
/// MP4A aka AAC: https://en.wikipedia.org/wiki/Advanced_Audio_Coding
Mp4a,
/// Opus: https://en.wikipedia.org/wiki/Opus_(audio_format)
Opus
Opus,
}
/// The video file format
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum VideoFormat {
#[serde(rename = "3gp")]
ThreeGp,
Mp4,
Webm,
}
impl FileFormat for VideoFormat {
fn extension(&self) -> &str {
match self {
VideoFormat::ThreeGp => ".3gp",
VideoFormat::Mp4 => ".mp4",
VideoFormat::Webm => ".webm",
}
}
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum AudioFormat {
M4a,
Webm,
}
impl FileFormat for AudioFormat {
fn extension(&self) -> &str {
match self {
AudioFormat::M4a => ".m4a",
AudioFormat::Webm => ".webm",
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]

View file

@ -1,2 +1,3 @@
pub mod range;
pub mod text;
// pub mod renderer;

View file

@ -1,5 +1,5 @@
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_with::{DeserializeAs, json::JsonString, serde_as, SerializeAs};
use serde_with::{json::JsonString, serde_as, DeserializeAs, SerializeAs};
#[serde_as]
#[derive(Deserialize, Serialize)]
@ -12,16 +12,25 @@ pub struct Range {
impl<'de> DeserializeAs<'de, std::ops::Range<u32>> for Range {
fn deserialize_as<D>(deserializer: D) -> Result<std::ops::Range<u32>, D::Error>
where
D: Deserializer<'de> {
where
D: Deserializer<'de>,
{
let range = Range::deserialize(deserializer)?;
Ok(std::ops::Range { start: range.start, end: range.end })
Ok(std::ops::Range {
start: range.start,
end: range.end,
})
}
}
impl SerializeAs<std::ops::Range<u32>> for Range {
fn serialize_as<S>(&std::ops::Range { start, end }: &std::ops::Range<u32>, serializer: S) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error> where
S: Serializer {
fn serialize_as<S>(
&std::ops::Range { start, end }: &std::ops::Range<u32>,
serializer: S,
) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error>
where
S: Serializer,
{
Range { start, end }.serialize(serializer)
}
}

View file

@ -0,0 +1,98 @@
use std::marker::PhantomData;
use serde::{de::Visitor, Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs, rust::maps_duplicate_key_is_error::deserialize};
/// ```json
/// {
/// itemSectionRenderer": {
/// "contents": [
/// {
/// "playlistVideoListRenderer": {
/// "contents": [
/// {
/// "playlistVideoRenderer": { ... }
/// },
/// {
/// "playlistVideoRenderer": { ... }
/// },
/// }
/// }
/// }
/// ]
/// }
/// }
/// ```
///
/// Renderer names:
///
/// 1 content element:
/// - tabRenderer > content
///
/// 1 content element (array):
/// - twoColumnBrowseResultsRenderer > tabs
/// - sectionListRenderer > contents
/// - itemSectionRenderer > contents
///
/// n content elements:
/// - playlistVideoListRenderer > contents
#[serde_as]
#[derive(Deserialize)]
#[serde(untagged, bound = "for<'de2> T: Deserialize<'de2>")]
pub enum Renderer<T> where for<'de2> T: Deserialize<'de2> {
Single {
#[serde_as(as = "crate::serializer::renderer::Renderer<T>")]
content: T,
},
Multiple {
#[serde(alias = "tabs")]
#[serde_as(as = "crate::serializer::renderer::Renderer<T>")]
contents: Vec<T>,
},
Content {
#[serde(flatten)]
inner: T,
},
}
// pub struct Renderer<T>(PhantomData<T>);
impl<'de, T> DeserializeAs<'de, T> for Renderer<T> {
fn deserialize_as<D>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
{
todo!()
}
}
impl<'de, T> DeserializeAs<'de, Vec<T>> for Renderer<T> {
fn deserialize_as<D>(deserializer: D) -> Result<Vec<T>, D::Error>
where
D: Deserializer<'de>,
{
todo!()
}
}
/*
struct RendererVisitor<T, U>(PhantomData<T>, PhantomData<U>);
impl<'de, T, U> Visitor<'de> for RendererVisitor<T, U>
where
U: DeserializeAs<'de, T>,
{
type Value = T;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a yt renderer")
}
fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: Deserializer<'de>, {
}
}
*/

View file

@ -8,7 +8,7 @@ const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
pub fn get_cg_from_regexes<'a, I>(mut regexes: I, text: &str, cg: usize) -> Option<String>
where
I: Iterator<Item = &'a Regex>,
{
{
regexes
.find_map(|pattern| pattern.captures(text).ok().flatten())
.map(|c| c.get(cg).unwrap().as_str().to_owned())
@ -21,9 +21,9 @@ pub fn random_string(charset: &[u8], length: usize) -> String {
unsafe {
for _ in 0..length {
result.push(
char::from(*charset.get_unchecked(rng.gen_range(0..charset.len())))
);
result.push(char::from(
*charset.get_unchecked(rng.gen_range(0..charset.len())),
));
}
}