Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: fetching comment count

Browse source
This commit is contained in:
ThetaDev 2022-09-20 17:24:16 +02:00
parent e800e16c68
commit 8c1e7bf6ac
8 changed files with 113 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cli/src/main.rs
View file

@ -150,7 +150,11 @@ async fn download_playlist(
let rp = RustyPipe::default();
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist.videos.extend_pages(rp.query(), usize::MAX).await.unwrap();
playlist
.videos
.extend_pages(rp.query(), usize::MAX)
.await
.unwrap();
// Indicatif setup
let multi = MultiProgress::new();

5
codegen/src/download_testfiles.rs
View file

@ -152,7 +152,10 @@ async fn comments_top(testfiles: &Path) {
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let rp = rp_testfile(&json_path);
rp.query().video_comments(&details.top_comments.ctoken.unwrap()).await.unwrap();
rp.query()
.video_comments(&details.top_comments.ctoken.unwrap())
.await
.unwrap();
// rp.query().video_comments("Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D").await.unwrap();
// Desktop 1

6
src/client/response/mod.rs
View file

@ -75,6 +75,12 @@ pub enum VideoListItem<T> {
None,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ContinuationItemRenderer {
pub continuation_endpoint: ContinuationEndpoint,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ContinuationEndpoint {

59
src/client/response/video_details.rs
View file

@ -12,8 +12,8 @@ use crate::serializer::{
};
use super::{
ChannelBadge, ContentsRenderer, ContinuationEndpoint, Icon, Thumbnails, VideoBadge,
VideoListItem, VideoOwner,
ChannelBadge, ContentsRenderer, ContinuationEndpoint, ContinuationItemRenderer, Icon,
Thumbnails, VideoBadge, VideoListItem, VideoOwner,
};
/*
@ -92,18 +92,11 @@ pub enum VideoResultsItem {
#[serde_as(deserialize_as = "DefaultOnError")]
metadata_row_container: Option<MetadataRowContainer>,
},
/*
/// The comment section consists of 2 ItemSectionRenderers:
///
/// 1. sectionIdentifier: "comments-entry-point", contains number of comments
/// 2. sectionIdentifier: "comment-item-section", contains continuation token
#[serde(rename_all = "camelCase")]
ItemSectionRenderer {
#[serde_as(as = "VecSkipError<_>")]
contents: Vec<ItemSection>,
section_identifier: String,
},
*/
ItemSectionRenderer(ItemSection),
#[serde(other, deserialize_with = "ignore_any")]
None,
}
@ -211,6 +204,47 @@ pub struct CurrentVideoWatchEndpoint {
pub video_id: String,
}
/// The comment section consists of 2 ItemSections:
///
/// 1. CommentsEntryPointHeaderRenderer: contains number of comments
/// 2. ContinuationItemRenderer: contains continuation token
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "sectionIdentifier")]
pub enum ItemSection {
CommentsEntryPoint {
#[serde_as(as = "VecSkipError<_>")]
contents: Vec<ItemSectionCommentCount>,
},
CommentItemSection {
#[serde_as(as = "VecSkipError<_>")]
contents: Vec<ItemSectionComments>,
},
}
/// Item section containing comment count
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ItemSectionCommentCount {
pub comments_entry_point_header_renderer: CommentsEntryPointHeaderRenderer,
}
/// Renderer of item section containing comment count
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CommentsEntryPointHeaderRenderer {
#[serde_as(as = "Text")]
pub comment_count: String,
}
/// Item section containing comments ctoken
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ItemSectionComments {
pub continuation_item_renderer: ContinuationItemRenderer,
}
/// Video recommendations
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
@ -461,10 +495,7 @@ pub enum CommentListItem {
rendering_priority: CommentPriority,
},
/// Reply comment
CommentRenderer {
#[serde(flatten)]
comment: CommentRenderer,
},
CommentRenderer(CommentRenderer),
/// Continuation token to fetch more comments
#[serde(rename_all = "camelCase")]
ContinuationItemRenderer {

72
src/client/video_details.rs
View file

@ -115,6 +115,9 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
let mut primary_info = None;
let mut secondary_info = None;
let mut comment_count_section = None;
let mut comment_ctoken_section = None;
primary_results.c.into_iter().for_each(|r| match r {
response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer { .. } => {
primary_info = Some(r);
@ -122,6 +125,16 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer { .. } => {
secondary_info = Some(r);
}
response::video_details::VideoResultsItem::ItemSectionRenderer(section) => {
match section {
response::video_details::ItemSection::CommentsEntryPoint { mut contents } => {
comment_count_section = contents.try_swap_remove(0);
}
response::video_details::ItemSection::CommentItemSection { mut contents } => {
comment_ctoken_section = contents.try_swap_remove(0);
}
}
}
response::video_details::VideoResultsItem::None => {}
});
@ -153,9 +166,21 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
_ => bail!("could not find primary_info"),
};
if publish_date.is_none() {
warnings.push(format!("could not parse date: {}", publish_date_txt));
}
/*
TODO: use large number parser for this
let comment_count = comment_count_section.and_then(|s| {
util::parse_numeric_or_warn::<u32>(
&s.comments_entry_point_header_renderer.comment_count,
&mut warnings,
)
});*/
let comment_ctoken = comment_ctoken_section.map(|s| {
s.continuation_item_renderer
.continuation_endpoint
.continuation_command
.token
});
let (owner, description, is_ccommons) = match secondary_info {
Some(response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer {
@ -220,23 +245,16 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
response::video_details::EngagementPanelRenderer::None => {},
});
let (top_comments_ctoken, latest_comments_ctoken) = match comment_panel {
Some(comments) => {
let mut items = comments
.engagement_panel_title_header_renderer
.menu
.sort_filter_sub_menu_renderer
.sub_menu_items;
let latest = items.try_swap_remove(1);
let top = items.try_swap_remove(0);
(
top.map(|c| c.service_endpoint.continuation_command.token),
latest.map(|c| c.service_endpoint.continuation_command.token),
)
}
None => (None, None),
};
let latest_comments_ctoken = comment_panel.and_then(|comments| {
let mut items = comments
.engagement_panel_title_header_renderer
.menu
.sort_filter_sub_menu_renderer
.sub_menu_items;
items
.try_swap_remove(1)
.map(|c| c.service_endpoint.continuation_command.token)
});
Ok(MapResult {
c: VideoDetails {
@ -258,12 +276,14 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
is_ccommons,
recommended: recommended.c,
top_comments: Paginator {
ctoken: top_comments_ctoken,
..Default::default()
count: None,
items: Vec::new(),
ctoken: comment_ctoken,
},
latest_comments: Paginator {
count: None,
items: Vec::new(),
ctoken: latest_comments_ctoken,
..Default::default()
},
},
warnings,
@ -325,7 +345,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
comments.push(res.c);
warnings.append(&mut res.warnings)
}
response::video_details::CommentListItem::CommentRenderer { comment } => {
response::video_details::CommentListItem::CommentRenderer(comment) => {
let mut res = map_comment(
comment,
None,
@ -439,7 +459,7 @@ fn map_comment(
.contents
.into_iter()
.filter_map(|item| match item {
response::video_details::CommentListItem::CommentRenderer { comment } => {
response::video_details::CommentListItem::CommentRenderer(comment) => {
let mut res = map_comment(
comment,
None,
@ -542,7 +562,7 @@ mod tests {
#[test_log::test(tokio::test)]
async fn get_video_comments() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("3pv_rHKnwAs").await.unwrap();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let rec = rp
.query()
.video_comments(&details.top_comments.ctoken.unwrap())

5
src/download.rs
View file

@ -269,8 +269,9 @@ pub async fn download_video(
let download_dir = PathBuf::from(output_dir);
let title = player_data.details.title.to_owned();
let output_fname_set = output_fname.is_some();
let output_fname = output_fname
.unwrap_or_else(|| filenamify::filenamify(format!("{} [{}]", title, player_data.details.id)));
let output_fname = output_fname.unwrap_or_else(|| {
filenamify::filenamify(format!("{} [{}]", title, player_data.details.id))
});
// Select streams to download
let (video, audio) = player_data.select_video_audio_stream(filter);

1
src/model/paginator.rs
View file

@ -1,6 +1,5 @@
use serde::{Deserialize, Serialize};
/// The paginator is a wrapper around a list of items that are fetched
/// in pages from the YouTube API (e.g. playlist items,
/// video recommendations or comments).

5
src/serializer/text.rs
View file

@ -253,7 +253,10 @@ impl TryFrom<TextLink> for crate::model::ChannelId {
page_type,
browse_id,
} => match page_type {
PageType::Channel => Ok(crate::model::ChannelId { id: browse_id, name: text }),
PageType::Channel => Ok(crate::model::ChannelId {
id: browse_id,
name: text,
}),
_ => Err(anyhow!("invalid channel link type")),
},
_ => Err(anyhow!("invalid channel link")),