From 972288d810923f2fc2f0d4aef522426e9f7703d5 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 19 Sep 2022 00:08:37 +0200 Subject: [PATCH] feat: add video details response model - add paginator, impl for playlist items - small model refactor - add ignore_any deserializer - removed unnecessary clones in response mapping --- cli/src/main.rs | 6 +- codegen/src/collect_playlist_dates.rs | 2 +- codegen/src/download_testfiles.rs | 63 +- src/client/mod.rs | 8 +- src/client/player.rs | 79 +- src/client/playlist.rs | 292 +- src/client/response/mod.rs | 33 +- src/client/response/playlist.rs | 7 +- .../response/{video.rs => video_details.rs} | 319 +- ...layer__tests__map_player_data_android.snap | 2 +- ...layer__tests__map_player_data_desktop.snap | 2 +- ...__tests__map_player_data_desktopmusic.snap | 2 +- ...t__player__tests__map_player_data_ios.snap | 2 +- ...__tests__map_player_data_tvhtml5embed.snap | 2 +- ...aylist__tests__map_playlist_data_long.snap | 3803 ++-- ...ist__tests__map_playlist_data_nomusic.snap | 2511 +-- ...ylist__tests__map_playlist_data_short.snap | 3689 ++-- src/client/video_details.rs | 112 + src/download.rs | 4 +- src/error.rs | 12 +- src/lib.rs | 1 + src/model/mod.rs | 94 +- src/model/paginator.rs | 54 + src/serializer/mod.rs | 58 + src/serializer/text.rs | 55 +- src/util.rs | 32 + testfiles/player_model/hdr.json | 2 +- testfiles/player_model/multilanguage.json | 2 +- .../video_details/video_details_ccommons.json | 12411 +++++++++++ .../video_details/video_details_chapters.json | 18060 ++++++++++++++++ .../video_details/video_details_music.json | 12443 +++++++++++ testfiles/video_details/video_details_mv.json | 12945 +++++++++++ 32 files changed, 61791 insertions(+), 5316 deletions(-) rename src/client/response/{video.rs => video_details.rs} (52%) create mode 100644 src/client/video_details.rs create mode 100644 src/model/paginator.rs create mode 100644 testfiles/video_details/video_details_ccommons.json create mode 100644 testfiles/video_details/video_details_chapters.json create mode 100644 testfiles/video_details/video_details_music.json create mode 100644 testfiles/video_details/video_details_mv.json diff --git a/cli/src/main.rs b/cli/src/main.rs index e6a77c0..e92ec9a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -60,7 +60,7 @@ async fn download_single_video( let res = async { let player_data = rp .query() - .get_player(video_id.as_str(), ClientType::TvHtml5Embed) + .player(video_id.as_str(), ClientType::TvHtml5Embed) .await .context(format!( "Failed to fetch player data for video {}", @@ -89,7 +89,7 @@ async fn download_single_video( .await .context(format!( "Failed to download video '{}' [{}]", - player_data.info.title, video_id + player_data.details.title, video_id )) } .await; @@ -149,7 +149,7 @@ async fn download_playlist( .expect("unable to build the HTTP client"); let rp = RustyPipe::default(); - let playlist = rp.query().get_playlist(id).await.unwrap(); + let playlist = rp.query().playlist(id).await.unwrap(); // Indicatif setup let multi = MultiProgress::new(); diff --git a/codegen/src/collect_playlist_dates.rs b/codegen/src/collect_playlist_dates.rs index 23b7f69..714487c 100644 --- a/codegen/src/collect_playlist_dates.rs +++ b/codegen/src/collect_playlist_dates.rs @@ -93,7 +93,7 @@ pub async fn collect_dates(project_root: &Path, concurrency: usize) { let mut map: BTreeMap = BTreeMap::new(); for (case, pl_id) in cases { - let playlist = rp.query().lang(lang).get_playlist(pl_id).await.unwrap(); + let playlist = rp.query().lang(lang).playlist(pl_id).await.unwrap(); map.insert(case, playlist.last_update_txt.unwrap()); } diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 58d4910..249b84e 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -8,6 +8,18 @@ use rustypipe::{ report::{Report, Reporter}, }; +pub async fn download_testfiles(project_root: &Path) { + let mut testfiles = project_root.to_path_buf(); + testfiles.push("testfiles"); + + tokio::join!( + player(&testfiles), + player_model(&testfiles), + playlist(&testfiles), + video_details(&testfiles), + ); +} + const CLIENT_TYPES: [ClientType; 5] = [ ClientType::Desktop, ClientType::DesktopMusic, @@ -31,6 +43,10 @@ impl TestFileReporter { impl Reporter for TestFileReporter { fn report(&self, report: &Report) { + let mut root = self.path.clone(); + root.set_file_name(""); + std::fs::create_dir_all(root).unwrap(); + let data = serde_json::from_str::(&report.http_request.resp_body).unwrap(); let file = File::create(&self.path).unwrap(); @@ -49,17 +65,6 @@ fn rp_testfile(json_path: &Path) -> RustyPipe { .build() } -pub async fn download_testfiles(project_root: &Path) { - let mut testfiles = project_root.to_path_buf(); - testfiles.push("testfiles"); - - tokio::join!( - player(&testfiles), - player_model(&testfiles), - playlist(&testfiles) - ); -} - async fn player(testfiles: &Path) { let video_id = "pPvd8UxmSbQ"; @@ -73,7 +78,7 @@ async fn player(testfiles: &Path) { } let rp = rp_testfile(&json_path); - rp.query().get_player(video_id, client_type).await.unwrap(); + rp.query().player(video_id, client_type).await.unwrap(); } } @@ -89,11 +94,7 @@ async fn player_model(testfiles: &Path) { continue; } - let player_data = rp - .query() - .get_player(id, ClientType::Desktop) - .await - .unwrap(); + let player_data = rp.query().player(id, ClientType::Desktop).await.unwrap(); let file = File::create(&json_path).unwrap(); serde_json::to_writer_pretty(file, &player_data).unwrap(); @@ -115,6 +116,32 @@ async fn playlist(testfiles: &Path) { } let rp = rp_testfile(&json_path); - rp.query().get_playlist(id).await.unwrap(); + rp.query().playlist(id).await.unwrap(); } } + + +async fn video_details(testfiles: &Path) { + for (name, id) in [ + ("music", "MZOgTu2dMTg"), + ("mv", "ZeerrnuLi5E"), + ("ccommons", "0rb9CfOvojk"), + ("chapters", "nFDBxBUfE74"), + ] { + let mut json_path = testfiles.to_path_buf(); + json_path.push("video_details"); + json_path.push(format!("video_details_{}.json", name)); + println!("{}", json_path.display()); + if json_path.exists() { + continue; + } + + let rp = rp_testfile(&json_path); + rp.query().video_details(id).await.unwrap(); + } +} + +#[tokio::test] +async fn x() { + video_details(Path::new("../testfiles")).await; +} diff --git a/src/client/mod.rs b/src/client/mod.rs index 4db6509..424a8b0 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1,5 +1,6 @@ pub mod player; pub mod playlist; +pub mod video_details; mod response; @@ -12,9 +13,7 @@ use fancy_regex::Regex; use log::{error, warn}; use once_cell::sync::Lazy; use rand::Rng; -use reqwest::{ - header, Client, ClientBuilder, Method, Request, RequestBuilder, Response, StatusCode, -}; +use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::Mutex; @@ -315,7 +314,7 @@ impl RustyPipeBuilder { rand::thread_rng().gen_range(100..1000) ), cache: Mutex::new(cache), - default_opts: RustyPipeOpts::default(), + default_opts: self.default_opts, }), } } @@ -455,6 +454,7 @@ impl RustyPipe { if status.is_success() || !status.is_server_error() { return res; } + // TODO: handle 429 (captcha) status.to_string() } Err(e) => { diff --git a/src/client/player.rs b/src/client/player.rs index c91f2b9..2fd4760 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -13,8 +13,8 @@ use serde::Serialize; use crate::{ deobfuscate::Deobfuscator, model::{ - AudioCodec, AudioFormat, AudioStream, AudioTrack, Channel, Language, Subtitle, VideoCodec, - VideoFormat, VideoInfo, VideoPlayer, VideoStream, + AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Language, Subtitle, + VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream, }, util, }; @@ -58,7 +58,7 @@ struct QContentPlaybackContext { } impl RustyPipeQuery { - pub async fn get_player(self, video_id: &str, client_type: ClientType) -> Result { + pub async fn player(self, video_id: &str, client_type: ClientType) -> Result { let q1 = self.clone(); let t_context = tokio::spawn(async move { q1.get_context(client_type, false).await }); let q2 = self.client.clone(); @@ -95,7 +95,7 @@ impl RustyPipeQuery { self.execute_request_deobf::( client_type, - "get_player", + "player", video_id, Method::POST, "player", @@ -169,13 +169,13 @@ impl MapResponse for response::Player { ); } - let video_info = VideoInfo { + let video_info = VideoPlayerDetails { id: video_details.video_id, title: video_details.title, description: video_details.short_description, length: video_details.length_seconds, thumbnails: video_details.thumbnail.unwrap_or_default().into(), - channel: Channel { + channel: ChannelId { id: video_details.channel_id, name: video_details.author, }, @@ -196,7 +196,7 @@ impl MapResponse for response::Player { warnings.append(&mut streaming_data.formats.warnings); warnings.append(&mut streaming_data.adaptive_formats.warnings); - let mut last_nsig: [String; 2] = ["".to_owned(), "".to_owned()]; + let mut last_nsig: [String; 2] = [String::new(), String::new()]; let mut video_streams: Vec = Vec::new(); let mut video_only_streams: Vec = Vec::new(); @@ -237,23 +237,26 @@ impl MapResponse for response::Player { video_only_streams.sort(); audio_streams.sort(); - let mut subtitles = vec![]; - if let Some(captions) = self.captions { - for c in captions.player_captions_tracklist_renderer.caption_tracks { - let lang_auto = c.name.strip_suffix(" (auto-generated)"); - - subtitles.push(Subtitle { - url: c.base_url, - lang: c.language_code, - lang_name: lang_auto.unwrap_or(&c.name).to_owned(), - auto_generated: lang_auto.is_some(), + let subtitles = self.captions.map_or(Vec::new(), |captions| { + captions + .player_captions_tracklist_renderer + .caption_tracks + .into_iter() + .map(|c| { + let lang_auto = c.name.strip_suffix(" (auto-generated)"); + Subtitle { + url: c.base_url, + lang: c.language_code, + lang_name: lang_auto.unwrap_or(&c.name).to_owned(), + auto_generated: lang_auto.is_some(), + } }) - } - } + .collect() + }); Ok(MapResult { c: VideoPlayer { - info: video_info, + details: video_info, video_streams, video_only_streams, audio_streams, @@ -599,7 +602,7 @@ mod tests { ); let is_desktop = name == "desktop" || name == "desktopmusic"; insta::assert_yaml_snapshot!(format!("map_player_data_{}", name), map_res.c, { - ".info.publish_date" => insta::dynamic_redaction(move |value, _path| { + ".details.publish_date" => insta::dynamic_redaction(move |value, _path| { if is_desktop { assert!(value.as_str().unwrap().starts_with("2019-05-30T00:00:00")); "2019-05-30T00:00:00" @@ -632,40 +635,36 @@ mod tests { #[test_log::test(tokio::test)] async fn t_get_player(#[case] client_type: ClientType) { let rp = RustyPipe::builder().strict().build(); - let player_data = rp - .query() - .get_player("n4tK7LYFxI0", client_type) - .await - .unwrap(); + let player_data = rp.query().player("n4tK7LYFxI0", client_type).await.unwrap(); // dbg!(&player_data); - assert_eq!(player_data.info.id, "n4tK7LYFxI0"); - assert_eq!(player_data.info.title, "Spektrem - Shine [NCS Release]"); + assert_eq!(player_data.details.id, "n4tK7LYFxI0"); + assert_eq!(player_data.details.title, "Spektrem - Shine [NCS Release]"); if client_type == ClientType::DesktopMusic { - assert!(player_data.info.description.is_none()); + assert!(player_data.details.description.is_none()); } else { - assert!(player_data.info.description.unwrap().starts_with( + assert!(player_data.details.description.unwrap().starts_with( "NCS (NoCopyrightSounds): Empowering Creators through Copyright / Royalty Free Music" )); } - assert_eq!(player_data.info.length, 259); - assert!(!player_data.info.thumbnails.is_empty()); - assert_eq!(player_data.info.channel.id, "UC_aEa8K-EOJ3D6gOs7HcyNg"); - assert_eq!(player_data.info.channel.name, "NoCopyrightSounds"); - assert!(player_data.info.view_count > 146818808); - assert_eq!(player_data.info.keywords[0], "spektrem"); - assert_eq!(player_data.info.is_live_content, false); + assert_eq!(player_data.details.length, 259); + assert!(!player_data.details.thumbnails.is_empty()); + assert_eq!(player_data.details.channel.id, "UC_aEa8K-EOJ3D6gOs7HcyNg"); + assert_eq!(player_data.details.channel.name, "NoCopyrightSounds"); + assert!(player_data.details.view_count > 146818808); + assert_eq!(player_data.details.keywords[0], "spektrem"); + assert_eq!(player_data.details.is_live_content, false); if client_type == ClientType::Desktop || client_type == ClientType::DesktopMusic { assert!(player_data - .info + .details .publish_date .unwrap() .to_string() .starts_with("2013-05-05 00:00:00")); - assert_eq!(player_data.info.category.unwrap(), "Music"); - assert_eq!(player_data.info.is_family_safe.unwrap(), true); + assert_eq!(player_data.details.category.unwrap(), "Music"); + assert_eq!(player_data.details.is_family_safe.unwrap(), true); } if client_type == ClientType::Ios { diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 9056ab7..306f41b 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -4,7 +4,7 @@ use serde::Serialize; use crate::{ deobfuscate::Deobfuscator, - model::{Channel, Language, Playlist, Thumbnail, Video}, + model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo}, serializer::text::{PageType, TextLink}, timeago, util, }; @@ -26,7 +26,7 @@ struct QPlaylistCont { } impl RustyPipeQuery { - pub async fn get_playlist(self, playlist_id: &str) -> Result { + pub async fn playlist(self, playlist_id: &str) -> Result { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QPlaylist { context, @@ -35,7 +35,7 @@ impl RustyPipeQuery { self.execute_request::( ClientType::Desktop, - "get_playlist", + "playlist", playlist_id, Method::POST, "browse", @@ -44,37 +44,22 @@ impl RustyPipeQuery { .await } - pub async fn get_playlist_cont(self, playlist: &mut Playlist) -> Result<()> { - match &playlist.ctoken { - Some(ctoken) => { - let context = self.get_context(ClientType::Desktop, true).await; - let request_body = QPlaylistCont { - context, - continuation: ctoken.to_owned(), - }; + pub async fn get_playlist_continuation(self, ctoken: &str) -> Result> { + let context = self.get_context(ClientType::Desktop, true).await; + let request_body = QPlaylistCont { + context, + continuation: ctoken.to_owned(), + }; - let (mut videos, ctoken) = self - .execute_request::( - ClientType::Desktop, - "get_playlist_cont", - &playlist.id, - Method::POST, - "browse", - &request_body, - ) - .await?; - - playlist.videos.append(&mut videos); - playlist.ctoken = ctoken; - - if playlist.ctoken.is_none() { - playlist.n_videos = playlist.videos.len() as u32; - } - - Ok(()) - } - None => Err(anyhow!("no ctoken")), - } + self.execute_request::( + ClientType::Desktop, + "get_playlist_continuation", + ctoken, + Method::POST, + "browse", + &request_body, + ) + .await } } @@ -85,85 +70,72 @@ impl MapResponse for response::Playlist { lang: Language, _deobf: Option<&Deobfuscator>, ) -> Result> { - let video_items = &some_or_bail!( - some_or_bail!( - some_or_bail!( - self.contents - .two_column_browse_results_renderer - .contents - .get(0), - Err(anyhow!("twoColumnBrowseResultsRenderer empty")) + // TODO: think about a deserializer that deserializes only first list item + let mut tcbr_contents = self.contents.two_column_browse_results_renderer.contents; + let video_items = some_or_bail!( + util::vec_try_swap_remove( + &mut some_or_bail!( + util::vec_try_swap_remove( + &mut some_or_bail!( + util::vec_try_swap_remove(&mut tcbr_contents, 0), + Err(anyhow!("twoColumnBrowseResultsRenderer empty")) + ) + .tab_renderer + .content + .section_list_renderer + .contents, + 0, + ), + Err(anyhow!("sectionListRenderer empty")) ) - .tab_renderer - .content - .section_list_renderer - .contents - .get(0), - Err(anyhow!("sectionListRenderer empty")) - ) - .item_section_renderer - .contents - .get(0), + .item_section_renderer + .contents, + 0 + ), Err(anyhow!("itemSectionRenderer empty")) ) .playlist_video_list_renderer .contents; - let (videos, ctoken) = map_playlist_items(&video_items.c); + let (videos, ctoken) = map_playlist_items(video_items.c); - let (thumbnails, last_update_txt) = match &self.sidebar { + let (thumbnails, last_update_txt) = match self.sidebar { Some(sidebar) => { - let primary = some_or_bail!( - sidebar.playlist_sidebar_renderer.items.get(0), + let mut sidebar_items = sidebar.playlist_sidebar_renderer.items; + let mut primary = some_or_bail!( + util::vec_try_swap_remove(&mut sidebar_items, 0), Err(anyhow!("no primary sidebar")) ); ( - &primary + primary .playlist_sidebar_primary_info_renderer .thumbnail_renderer .playlist_video_thumbnail_renderer - .thumbnail - .thumbnails, - primary - .playlist_sidebar_primary_info_renderer - .stats - .get(2) - .map(|t| t.to_owned()), + .thumbnail, + util::vec_try_swap_remove( + &mut primary.playlist_sidebar_primary_info_renderer.stats, + 2, + ), ) } None => { let header_banner = some_or_bail!( - &self.header.playlist_header_renderer.playlist_header_banner, + self.header.playlist_header_renderer.playlist_header_banner, Err(anyhow!("no thumbnail found")) ); - let last_update_txt = self - .header - .playlist_header_renderer - .byline - .get(1) - .map(|b| b.playlist_byline_renderer.text.to_owned()); + let mut byline = self.header.playlist_header_renderer.byline; + let last_update_txt = util::vec_try_swap_remove(&mut byline, 1) + .map(|b| b.playlist_byline_renderer.text); ( - &header_banner - .hero_playlist_thumbnail_renderer - .thumbnail - .thumbnails, + header_banner.hero_playlist_thumbnail_renderer.thumbnail, last_update_txt, ) } }; - let thumbnails = thumbnails - .iter() - .map(|t| Thumbnail { - url: t.url.to_owned(), - width: t.width, - height: t.height, - }) - .collect::>(); - let n_videos = match ctoken { Some(_) => { ok_or_bail!( @@ -187,14 +159,14 @@ impl MapResponse for response::Playlist { text, page_type: PageType::Channel, browse_id, - }) => Some(Channel { + }) => Some(ChannelId { id: browse_id, name: text, }), _ => None, }; - let mut warnings = video_items.warnings.to_owned(); + let mut warnings = video_items.warnings; let last_update = match &last_update_txt { Some(textual_date) => { let parsed = timeago::parse_textual_date_to_dt(lang, textual_date); @@ -210,10 +182,12 @@ impl MapResponse for response::Playlist { c: Playlist { id: playlist_id, name, - videos, + videos: Paginator { + items: videos, + ctoken, + }, n_videos, - ctoken, - thumbnails, + thumbnails: thumbnails.into(), description, channel, last_update, @@ -224,60 +198,52 @@ impl MapResponse for response::Playlist { } } -impl MapResponse<(Vec