From 8c1e7bf6ac9a2da4aeab54beca90dfb88cee992d Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Tue, 20 Sep 2022 17:24:16 +0200 Subject: [PATCH] fix: fetching comment count --- cli/src/main.rs | 6 ++- codegen/src/download_testfiles.rs | 5 +- src/client/response/mod.rs | 6 +++ src/client/response/video_details.rs | 59 +++++++++++++++++------ src/client/video_details.rs | 72 ++++++++++++++++++---------- src/download.rs | 5 +- src/model/paginator.rs | 1 - src/serializer/text.rs | 5 +- 8 files changed, 113 insertions(+), 46 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 27a8445..1b46831 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -150,7 +150,11 @@ async fn download_playlist( let rp = RustyPipe::default(); let mut playlist = rp.query().playlist(id).await.unwrap(); - playlist.videos.extend_pages(rp.query(), usize::MAX).await.unwrap(); + playlist + .videos + .extend_pages(rp.query(), usize::MAX) + .await + .unwrap(); // Indicatif setup let multi = MultiProgress::new(); diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index c91a6a2..09db50a 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -152,7 +152,10 @@ async fn comments_top(testfiles: &Path) { let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); let rp = rp_testfile(&json_path); - rp.query().video_comments(&details.top_comments.ctoken.unwrap()).await.unwrap(); + rp.query() + .video_comments(&details.top_comments.ctoken.unwrap()) + .await + .unwrap(); // rp.query().video_comments("Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D").await.unwrap(); // Desktop 1 diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index e73c8f3..6e2c2c4 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -75,6 +75,12 @@ pub enum VideoListItem { None, } +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ContinuationItemRenderer { + pub continuation_endpoint: ContinuationEndpoint, +} + #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ContinuationEndpoint { diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index 4ac2951..44d1b17 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -12,8 +12,8 @@ use crate::serializer::{ }; use super::{ - ChannelBadge, ContentsRenderer, ContinuationEndpoint, Icon, Thumbnails, VideoBadge, - VideoListItem, VideoOwner, + ChannelBadge, ContentsRenderer, ContinuationEndpoint, ContinuationItemRenderer, Icon, + Thumbnails, VideoBadge, VideoListItem, VideoOwner, }; /* @@ -92,18 +92,11 @@ pub enum VideoResultsItem { #[serde_as(deserialize_as = "DefaultOnError")] metadata_row_container: Option, }, - /* /// The comment section consists of 2 ItemSectionRenderers: /// /// 1. sectionIdentifier: "comments-entry-point", contains number of comments /// 2. sectionIdentifier: "comment-item-section", contains continuation token - #[serde(rename_all = "camelCase")] - ItemSectionRenderer { - #[serde_as(as = "VecSkipError<_>")] - contents: Vec, - section_identifier: String, - }, - */ + ItemSectionRenderer(ItemSection), #[serde(other, deserialize_with = "ignore_any")] None, } @@ -211,6 +204,47 @@ pub struct CurrentVideoWatchEndpoint { pub video_id: String, } +/// The comment section consists of 2 ItemSections: +/// +/// 1. CommentsEntryPointHeaderRenderer: contains number of comments +/// 2. ContinuationItemRenderer: contains continuation token +#[serde_as] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "sectionIdentifier")] +pub enum ItemSection { + CommentsEntryPoint { + #[serde_as(as = "VecSkipError<_>")] + contents: Vec, + }, + CommentItemSection { + #[serde_as(as = "VecSkipError<_>")] + contents: Vec, + }, +} + +/// Item section containing comment count +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ItemSectionCommentCount { + pub comments_entry_point_header_renderer: CommentsEntryPointHeaderRenderer, +} + +/// Renderer of item section containing comment count +#[serde_as] +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CommentsEntryPointHeaderRenderer { + #[serde_as(as = "Text")] + pub comment_count: String, +} + +/// Item section containing comments ctoken +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ItemSectionComments { + pub continuation_item_renderer: ContinuationItemRenderer, +} + /// Video recommendations #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] @@ -461,10 +495,7 @@ pub enum CommentListItem { rendering_priority: CommentPriority, }, /// Reply comment - CommentRenderer { - #[serde(flatten)] - comment: CommentRenderer, - }, + CommentRenderer(CommentRenderer), /// Continuation token to fetch more comments #[serde(rename_all = "camelCase")] ContinuationItemRenderer { diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 1dac90c..0fabe7e 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -115,6 +115,9 @@ impl MapResponse for response::VideoDetails { let mut primary_info = None; let mut secondary_info = None; + let mut comment_count_section = None; + let mut comment_ctoken_section = None; + primary_results.c.into_iter().for_each(|r| match r { response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer { .. } => { primary_info = Some(r); @@ -122,6 +125,16 @@ impl MapResponse for response::VideoDetails { response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer { .. } => { secondary_info = Some(r); } + response::video_details::VideoResultsItem::ItemSectionRenderer(section) => { + match section { + response::video_details::ItemSection::CommentsEntryPoint { mut contents } => { + comment_count_section = contents.try_swap_remove(0); + } + response::video_details::ItemSection::CommentItemSection { mut contents } => { + comment_ctoken_section = contents.try_swap_remove(0); + } + } + } response::video_details::VideoResultsItem::None => {} }); @@ -153,9 +166,21 @@ impl MapResponse for response::VideoDetails { _ => bail!("could not find primary_info"), }; - if publish_date.is_none() { - warnings.push(format!("could not parse date: {}", publish_date_txt)); - } + /* + TODO: use large number parser for this + let comment_count = comment_count_section.and_then(|s| { + util::parse_numeric_or_warn::( + &s.comments_entry_point_header_renderer.comment_count, + &mut warnings, + ) + });*/ + + let comment_ctoken = comment_ctoken_section.map(|s| { + s.continuation_item_renderer + .continuation_endpoint + .continuation_command + .token + }); let (owner, description, is_ccommons) = match secondary_info { Some(response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer { @@ -220,23 +245,16 @@ impl MapResponse for response::VideoDetails { response::video_details::EngagementPanelRenderer::None => {}, }); - let (top_comments_ctoken, latest_comments_ctoken) = match comment_panel { - Some(comments) => { - let mut items = comments - .engagement_panel_title_header_renderer - .menu - .sort_filter_sub_menu_renderer - .sub_menu_items; - let latest = items.try_swap_remove(1); - let top = items.try_swap_remove(0); - - ( - top.map(|c| c.service_endpoint.continuation_command.token), - latest.map(|c| c.service_endpoint.continuation_command.token), - ) - } - None => (None, None), - }; + let latest_comments_ctoken = comment_panel.and_then(|comments| { + let mut items = comments + .engagement_panel_title_header_renderer + .menu + .sort_filter_sub_menu_renderer + .sub_menu_items; + items + .try_swap_remove(1) + .map(|c| c.service_endpoint.continuation_command.token) + }); Ok(MapResult { c: VideoDetails { @@ -258,12 +276,14 @@ impl MapResponse for response::VideoDetails { is_ccommons, recommended: recommended.c, top_comments: Paginator { - ctoken: top_comments_ctoken, - ..Default::default() + count: None, + items: Vec::new(), + ctoken: comment_ctoken, }, latest_comments: Paginator { + count: None, + items: Vec::new(), ctoken: latest_comments_ctoken, - ..Default::default() }, }, warnings, @@ -325,7 +345,7 @@ impl MapResponse> for response::VideoComments { comments.push(res.c); warnings.append(&mut res.warnings) } - response::video_details::CommentListItem::CommentRenderer { comment } => { + response::video_details::CommentListItem::CommentRenderer(comment) => { let mut res = map_comment( comment, None, @@ -439,7 +459,7 @@ fn map_comment( .contents .into_iter() .filter_map(|item| match item { - response::video_details::CommentListItem::CommentRenderer { comment } => { + response::video_details::CommentListItem::CommentRenderer(comment) => { let mut res = map_comment( comment, None, @@ -542,7 +562,7 @@ mod tests { #[test_log::test(tokio::test)] async fn get_video_comments() { let rp = RustyPipe::builder().strict().build(); - let details = rp.query().video_details("3pv_rHKnwAs").await.unwrap(); + let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap(); let rec = rp .query() .video_comments(&details.top_comments.ctoken.unwrap()) diff --git a/src/download.rs b/src/download.rs index 5b427c8..80c405c 100644 --- a/src/download.rs +++ b/src/download.rs @@ -269,8 +269,9 @@ pub async fn download_video( let download_dir = PathBuf::from(output_dir); let title = player_data.details.title.to_owned(); let output_fname_set = output_fname.is_some(); - let output_fname = output_fname - .unwrap_or_else(|| filenamify::filenamify(format!("{} [{}]", title, player_data.details.id))); + let output_fname = output_fname.unwrap_or_else(|| { + filenamify::filenamify(format!("{} [{}]", title, player_data.details.id)) + }); // Select streams to download let (video, audio) = player_data.select_video_audio_stream(filter); diff --git a/src/model/paginator.rs b/src/model/paginator.rs index c1182b8..ad1e331 100644 --- a/src/model/paginator.rs +++ b/src/model/paginator.rs @@ -1,6 +1,5 @@ use serde::{Deserialize, Serialize}; - /// The paginator is a wrapper around a list of items that are fetched /// in pages from the YouTube API (e.g. playlist items, /// video recommendations or comments). diff --git a/src/serializer/text.rs b/src/serializer/text.rs index f3ed430..3cc5e61 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -253,7 +253,10 @@ impl TryFrom for crate::model::ChannelId { page_type, browse_id, } => match page_type { - PageType::Channel => Ok(crate::model::ChannelId { id: browse_id, name: text }), + PageType::Channel => Ok(crate::model::ChannelId { + id: browse_id, + name: text, + }), _ => Err(anyhow!("invalid channel link type")), }, _ => Err(anyhow!("invalid channel link")),