Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add video details mapping

Browse source 
- TODO: fix fetching comment count
This commit is contained in:
ThetaDev 2022-09-20 16:14:57 +02:00
parent df6543d62e
commit e800e16c68
13 changed files with 38081 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

48
codegen/src/download_testfiles.rs
View file

@ -17,6 +17,7 @@ pub async fn download_testfiles(project_root: &Path) {
player_model(&testfiles),
playlist(&testfiles),
video_details(&testfiles),
comments_top(&testfiles),
);
}
@ -120,7 +121,6 @@ async fn playlist(testfiles: &Path) {
}
}
async fn video_details(testfiles: &Path) {
for (name, id) in [
("music", "MZOgTu2dMTg"),
@ -131,7 +131,6 @@ async fn video_details(testfiles: &Path) {
let mut json_path = testfiles.to_path_buf();
json_path.push("video_details");
json_path.push(format!("video_details_{}.json", name));
println!("{}", json_path.display());
if json_path.exists() {
continue;
}
@ -141,7 +140,46 @@ async fn video_details(testfiles: &Path) {
}
}
#[tokio::test]
async fn x() {
video_details(Path::new("../testfiles")).await;
async fn comments_top(testfiles: &Path) {
let mut json_path = testfiles.to_path_buf();
json_path.push("video_details");
json_path.push(format!("comments_top.json"));
if json_path.exists() {
return;
}
let rp = RustyPipe::new();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let rp = rp_testfile(&json_path);
rp.query().video_comments(&details.top_comments.ctoken.unwrap()).await.unwrap();
// rp.query().video_comments("Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D").await.unwrap();
// Desktop 1
// top: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// latest: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczABeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// shows count
// Desktop 2
// top: Eg0SC0lITnpPSGk4c0pzGAYyVSIuIgtJSE56T0hpOHNKczAAeAKqAhpVZ3lVZG5WQnBNR09tMnVMR3o1NEFhQUJBZzABQiFlbmdhZ2VtZW50LXBhbmVsLWNvbW1lbnRzLXNlY3Rpb24%3D
// shows no count
// latest: Eg0SC0lITnpPSGk4c0pzGAYyOCIRIgtJSE56T0hpOHNKczABeAIwAUIhZW5nYWdlbWVudC1wYW5lbC1jb21tZW50cy1zZWN0aW9u
// shows no count
// cont: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// shows count
}
#[tokio::test]
async fn test() {
let id = "IHNzOHi8sJs";
let rp = RustyPipe::new();
let details = rp.query().video_details(id).await.unwrap();
let ctoken_top = details.top_comments.ctoken;
let ctoken_latest = details.latest_comments.ctoken;
dbg!(ctoken_top);
dbg!(ctoken_latest);
}

11609
notes/next/next_comments_by_artist.json Normal file
View file

File diff suppressed because it is too large Load diff

101
src/client/playlist.rs
View file

@ -1,3 +1,5 @@
use std::convert::TryFrom;
use anyhow::{anyhow, bail, Result};
use reqwest::Method;
use serde::Serialize;
@ -5,7 +7,6 @@ use serde::Serialize;
use crate::{
deobfuscate::Deobfuscator,
model::{ChannelId, Language, Paginator, Playlist, PlaylistVideo},
serializer::text::{PageType, TextLink},
timeago,
util::{self, TryRemove},
};
@ -45,7 +46,7 @@ impl RustyPipeQuery {
.await
}
pub async fn get_playlist_continuation(self, ctoken: &str) -> Result<Paginator<PlaylistVideo>> {
pub async fn playlist_continuation(self, ctoken: &str) -> Result<Paginator<PlaylistVideo>> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QPlaylistCont {
context,
@ -151,41 +152,28 @@ impl MapResponse<Playlist> for response::Playlist {
let name = self.header.playlist_header_renderer.title;
let description = self.header.playlist_header_renderer.description_text;
let channel = match self.header.playlist_header_renderer.owner_text {
Some(TextLink::Browse {
text,
page_type: PageType::Channel,
browse_id,
}) => Some(ChannelId {
id: browse_id,
name: text,
}),
_ => None,
};
let channel = self
.header
.playlist_header_renderer
.owner_text
.and_then(|link| ChannelId::try_from(link).ok());
let mut warnings = video_items.warnings;
let last_update = match &last_update_txt {
Some(textual_date) => {
let parsed = timeago::parse_textual_date_to_dt(lang, textual_date);
if parsed.is_none() {
warnings.push(format!("could not parse textual date `{}`", textual_date));
}
parsed
}
None => None,
};
let last_update = last_update_txt
.as_ref()
.and_then(|txt| timeago::parse_textual_date_or_warn(lang, txt, &mut warnings));
Ok(MapResult {
c: Playlist {
id: playlist_id,
name,
videos: Paginator {
count: Some(n_videos),
items: videos,
ctoken,
},
n_videos,
thumbnails: thumbnails.into(),
video_count: n_videos,
thumbnail: thumbnails.into(),
description,
channel,
last_update,
@ -213,7 +201,11 @@ impl MapResponse<Paginator<PlaylistVideo>> for response::PlaylistCont {
map_playlist_items(action.append_continuation_items_action.continuation_items.c);
Ok(MapResult {
c: Paginator { items, ctoken },
c: Paginator {
count: None,
items,
ctoken,
},
warnings: action
.append_continuation_items_action
.continuation_items
@ -229,23 +221,18 @@ fn map_playlist_items(
let videos = items
.into_iter()
.filter_map(|it| match it {
response::VideoListItem::GridVideoRenderer { video } => match video.channel {
TextLink::Browse {
text,
page_type: PageType::Channel,
browse_id,
} => Some(PlaylistVideo {
id: video.video_id,
title: video.title,
length: video.length_seconds,
thumbnails: video.thumbnail.into(),
channel: ChannelId {
id: browse_id,
name: text,
},
}),
_ => None,
},
response::VideoListItem::GridVideoRenderer { video } => {
match ChannelId::try_from(video.channel) {
Ok(channel) => Some(PlaylistVideo {
id: video.video_id,
title: video.title,
length: video.length_seconds,
thumbnail: video.thumbnail.into(),
channel,
}),
Err(_) => None,
}
}
response::VideoListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
@ -261,7 +248,7 @@ fn map_playlist_items(
impl Paginator<PlaylistVideo> {
pub async fn next(&self, query: RustyPipeQuery) -> Result<Option<Self>> {
Ok(match &self.ctoken {
Some(ctoken) => Some(query.get_playlist_continuation(ctoken).await?),
Some(ctoken) => Some(query.playlist_continuation(ctoken).await?),
None => None,
})
}
@ -282,12 +269,8 @@ impl Paginator<PlaylistVideo> {
pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> {
for _ in 0..n_pages {
match self.extend(query.clone()).await {
Ok(false) => {
break;
}
Err(e) => {
return Err(e);
}
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
@ -297,12 +280,8 @@ impl Paginator<PlaylistVideo> {
pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> {
while self.items.len() < n_items {
match self.extend(query.clone()).await {
Ok(false) => {
break;
}
Err(e) => {
return Err(e);
}
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
@ -363,13 +342,13 @@ mod tests {
assert_eq!(playlist.name, name);
assert!(!playlist.videos.is_empty());
assert_eq!(!playlist.videos.is_exhausted(), is_long);
assert!(playlist.n_videos > 10);
assert_eq!(playlist.n_videos > 100, is_long);
assert!(playlist.video_count > 10);
assert_eq!(playlist.video_count > 100, is_long);
assert_eq!(playlist.description, description);
if channel.is_some() {
assert_eq!(playlist.channel, channel);
}
assert!(!playlist.thumbnails.is_empty());
assert!(!playlist.thumbnail.is_empty());
}
#[rstest]
@ -410,6 +389,7 @@ mod tests {
.await
.unwrap();
assert!(playlist.videos.items.len() > 100);
assert!(playlist.videos.count.unwrap() > 100);
}
#[test_log::test(tokio::test)]
@ -423,5 +403,6 @@ mod tests {
playlist.videos.extend_limit(rp.query(), 101).await.unwrap();
assert!(playlist.videos.items.len() > 100);
assert!(playlist.videos.count.unwrap() > 100);
}
}

85
src/client/response/mod.rs
View file

@ -68,6 +68,9 @@ pub enum VideoListItem<T> {
continuation_endpoint: ContinuationEndpoint,
},
/// No video list item (e.g. ad)
///
/// Note that there are sometimes playlists among the recommended
/// videos. They are currently ignored.
#[serde(other, deserialize_with = "ignore_any")]
None,
}
@ -84,10 +87,22 @@ pub struct ContinuationCommand {
pub token: String,
}
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Icon {
pub icon_type: String,
pub icon_type: IconType,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum IconType {
/// Checkmark for verified channels
Check,
/// Music note for verified artists
OfficialArtistBadge,
/// Like button
Like,
}
#[derive(Clone, Debug, Deserialize)]
@ -107,24 +122,24 @@ pub struct VideoOwnerRenderer {
pub subscriber_count_text: Option<String>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub badges: Vec<UserBadge>,
pub badges: Vec<ChannelBadge>,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct UserBadge {
pub metadata_badge_renderer: UserBadgeRenderer,
pub struct ChannelBadge {
pub metadata_badge_renderer: ChannelBadgeRenderer,
}
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct UserBadgeRenderer {
pub style: UserBadgeStyle,
pub struct ChannelBadgeRenderer {
pub style: ChannelBadgeStyle,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum UserBadgeStyle {
pub enum ChannelBadgeStyle {
BadgeStyleTypeVerified,
BadgeStyleTypeVerifiedArtist,
}
@ -155,6 +170,29 @@ pub enum TimeOverlayStyle {
Shorts,
}
/// Badges are displayed on the video thumbnail and
/// show certain video properties (e.g. active livestream)
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoBadge {
pub metadata_badge_renderer: VideoBadgeRenderer,
}
/// Badges are displayed on the video thumbnail and
/// show certain video properties (e.g. active livestream)
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoBadgeRenderer {
pub style: VideoBadgeStyle,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum VideoBadgeStyle {
/// Active livestream
BadgeStyleTypeLiveNow,
}
// YouTube Music
#[serde_as]
@ -247,3 +285,36 @@ impl From<Thumbnails> for Vec<crate::model::Thumbnail> {
.collect()
}
}
impl From<Vec<ChannelBadge>> for crate::model::Verification {
fn from(badges: Vec<ChannelBadge>) -> Self {
badges.get(0).map_or(crate::model::Verification::None, |b| {
match b.metadata_badge_renderer.style {
ChannelBadgeStyle::BadgeStyleTypeVerified => Self::Verified,
ChannelBadgeStyle::BadgeStyleTypeVerifiedArtist => Self::Artist,
}
})
}
}
impl From<Icon> for crate::model::Verification {
fn from(icon: Icon) -> Self {
match icon.icon_type {
IconType::Check => Self::Verified,
IconType::OfficialArtistBadge => Self::Artist,
_ => Self::None,
}
}
}
pub trait IsLive {
fn is_live(&self) -> bool;
}
impl IsLive for Vec<VideoBadge> {
fn is_live(&self) -> bool {
self.iter().any(|badge| {
badge.metadata_badge_renderer.style == VideoBadgeStyle::BadgeStyleTypeLiveNow
})
}
}

88
src/client/response/video_details.rs
View file

@ -11,7 +11,10 @@ use crate::serializer::{
VecLogError,
};
use super::{ContentsRenderer, ContinuationEndpoint, Icon, Thumbnails, VideoListItem, VideoOwner};
use super::{
ChannelBadge, ContentsRenderer, ContinuationEndpoint, Icon, Thumbnails, VideoBadge,
VideoListItem, VideoOwner,
};
/*
#VIDEO DETAILS
@ -24,6 +27,8 @@ use super::{ContentsRenderer, ContinuationEndpoint, Icon, Thumbnails, VideoListI
pub struct VideoDetails {
/// Video metadata + recommended videos
pub contents: Contents,
/// Video ID
pub current_video_endpoint: CurrentVideoEndpoint,
#[serde_as(as = "VecLogError<_>")]
/// Video chapters + comment section
pub engagement_panels: MapResult<Vec<EngagementPanel>>,
@ -113,6 +118,7 @@ pub struct ViewCount {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ViewCountRenderer {
/// View count (`232,975,196 views`)
#[serde_as(as = "Text")]
pub view_count: String,
}
@ -147,9 +153,9 @@ pub struct ToggleButtonWrap {
pub struct ToggleButton {
/// Icon type: `LIKE` / `DISLIKE`
pub default_icon: Icon,
/// Number of likes (`4,010,157 likes`)
/// Number of likes (`like this video along with 4,010,156 other people`)
#[serde_as(as = "AccessibilityText")]
pub default_text: String,
pub accessibility_data: String,
}
/// Shows additional video metadata. Its only known use is for
@ -192,22 +198,18 @@ pub struct MetadataRowRenderer {
pub contents: Vec<Vec<TextLink>>,
}
/*
#[serde_as]
/// Contains current video ID
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum ItemSection {
#[serde(rename_all = "camelCase")]
CommentsEntryPointHeaderRenderer {
#[serde_as(as = "Text")]
comment_count: String,
},
#[serde(rename_all = "camelCase")]
ContinuationItemRenderer {
continuation_endpoint: ContinuationEndpoint,
},
pub struct CurrentVideoEndpoint {
pub watch_endpoint: CurrentVideoWatchEndpoint,
}
/// Contains current video ID
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CurrentVideoWatchEndpoint {
pub video_id: String,
}
*/
/// Video recommendations
#[derive(Clone, Debug, Deserialize)]
@ -237,40 +239,25 @@ pub struct RecommendedVideo {
#[serde(rename = "shortBylineText")]
#[serde_as(as = "TextLink")]
pub channel: TextLink,
pub channel_thumbnail: Thumbnails,
/// Channel verification badge
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub owner_badges: Vec<ChannelBadge>,
#[serde_as(as = "Option<Text>")]
pub length_text: Option<String>,
/// (e.g. `11 months ago`)
#[serde_as(as = "Option<Text>")]
pub published_time_text: Option<String>,
#[serde_as(as = "Text")]
pub view_count_text: String,
/// Badges are displayed on the video thumbnail and
/// show certain video properties (e.g. active livestream)
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub badges: Vec<VideoBadge>,
}
/// Badges are displayed on the video thumbnail and
/// show certain video properties (e.g. active livestream)
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoBadge {
pub metadata_badge_renderer: VideoBadgeRenderer,
}
/// Badges are displayed on the video thumbnail and
/// show certain video properties (e.g. active livestream)
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VideoBadgeRenderer {
pub style: VideoBadgeStyle,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "SCREAMING_SNAKE_CASE")]
pub enum VideoBadgeStyle {
/// Active livestream
BadgeStyleTypeLiveNow,
}
/// The engagement panels are displayed below the video and contain chapter markers
/// and the comment section.
#[derive(Clone, Debug, Deserialize)]
@ -282,13 +269,13 @@ pub struct EngagementPanel {
/// The engagement panels are displayed below the video and contain chapter markers
/// and the comment section.
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "panelIdentifier")]
#[serde(rename_all = "kebab-case", tag = "targetId")]
pub enum EngagementPanelRenderer {
/// Chapter markers
EngagementPanelMacroMarkersDescriptionChapters { content: ChapterMarkersContent },
/// Comment section (contains no comments, but the
/// continuation tokens for fetching top/latest comments)
CommentItemSection { header: CommentItemSectionHeader },
EngagementPanelCommentsSection { header: CommentItemSectionHeader },
/// Ignored items:
/// - `engagement-panel-ads`
/// - `engagement-panel-structured-description`
@ -486,8 +473,9 @@ pub enum CommentListItem {
/// Header of the comment section (contains number of comments)
#[serde(rename_all = "camelCase")]
CommentsHeaderRenderer {
#[serde_as(as = "Text")]
count_text: Vec<String>
/// `4,238,993 Comments`
#[serde_as(as = "Option<Text>")]
count_text: Option<String>,
},
}
@ -520,11 +508,12 @@ pub struct CommentRenderer {
pub published_time_text: String,
pub comment_id: String,
pub author_is_channel_owner: bool,
#[serde_as(as = "Option<Text>")]
pub vote_count: Option<String>,
// #[serde_as(as = "Option<Text>")]
// pub vote_count: Option<String>,
pub author_comment_badge: Option<AuthorCommentBadge>,
#[serde(default)]
pub reply_count: u32,
/// Buttons for comment interaction (Like/Dislike/Reply)
pub action_buttons: CommentActionButtons,
}
@ -568,17 +557,20 @@ pub struct RepliesRenderer {
pub contents: Vec<CommentListItem>,
}
/// These are the buttons for comment interaction. Contains the CreatorHeart.
/// These are the buttons for comment interaction (Like/Dislike/Reply).
/// Contains the CreatorHeart.
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CommentActionButtons {
pub comment_action_buttons_renderer: CommentActionButtonsRenderer,
}
/// These are the buttons for comment interaction. Contains the CreatorHeart.
/// These are the buttons for comment interaction (Like/Dislike/Reply).
/// Contains the CreatorHeart.
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CommentActionButtonsRenderer {
pub like_button: ToggleButtonWrap,
pub creator_heart: Option<CreatorHeart>,
}
@ -607,5 +599,7 @@ pub struct AuthorCommentBadge {
#[serde(rename_all = "camelCase")]
pub struct AuthorCommentBadgeRenderer {
/// Verified: `CHECK`
///
/// Artist: `OFFICIAL_ARTIST_BADGE`
pub icon: Icon,
}

516
src/client/video_details.rs
View file

@ -1,10 +1,20 @@
use anyhow::Result;
use std::convert::TryFrom;
use anyhow::{anyhow, bail, Result};
use reqwest::Method;
use serde::Serialize;
use crate::{model::VideoDetails, serializer::MapResult};
use crate::{
model::{Channel, ChannelId, Comment, Language, Paginator, RecommendedVideo, VideoDetails},
serializer::MapResult,
timeago,
util::{self, TryRemove},
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
use super::{
response::{self, IconType, IsLive},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
#[derive(Clone, Debug, Serialize)]
struct QVideo {
@ -44,48 +54,41 @@ impl RustyPipeQuery {
.await
}
/*
async fn get_comments_response(&self, ctoken: &str) -> Result<response::VideoComments> {
let client = self.get_ytclient(ClientType::Desktop);
let context = client.get_context(true).await;
pub async fn video_recommendations(self, ctoken: &str) -> Result<Paginator<RecommendedVideo>> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QVideoCont {
context,
continuation: ctoken.to_owned(),
};
let resp = client
.request_builder(Method::POST, "next")
.await
.json(&request_body)
.send()
.await?
.error_for_status()?;
Ok(resp.json::<response::VideoComments>().await?)
self.execute_request::<response::VideoRecommendations, _, _>(
ClientType::Desktop,
"video_recommendations",
ctoken,
Method::POST,
"next",
&request_body,
)
.await
}
async fn get_recommendations_response(
&self,
ctoken: &str,
) -> Result<response::VideoRecommendations> {
let client = self.get_ytclient(ClientType::Desktop);
let context = client.get_context(true).await;
pub async fn video_comments(self, ctoken: &str) -> Result<Paginator<Comment>> {
let context = self.get_context(ClientType::Desktop, true).await;
let request_body = QVideoCont {
context,
continuation: ctoken.to_owned(),
};
let resp = client
.request_builder(Method::POST, "next")
.await
.json(&request_body)
.send()
.await?
.error_for_status()?;
Ok(resp.json::<response::VideoRecommendations>().await?)
self.execute_request::<response::VideoComments, _, _>(
ClientType::Desktop,
"video_comments",
ctoken,
Method::POST,
"next",
&request_body,
)
.await
}
*/
}
impl MapResponse<VideoDetails> for response::VideoDetails {
@ -95,18 +98,457 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<VideoDetails>> {
let mut warnings = Vec::new();
let video_id = self.current_video_endpoint.watch_endpoint.video_id;
if id != video_id {
bail!("got wrong playlist id {}, expected {}", video_id, id);
}
let mut primary_results = self
.contents
.two_column_watch_next_results
.results
.results
.contents;
warnings.append(&mut primary_results.warnings);
let mut primary_info = None;
let mut secondary_info = None;
primary_results.c.into_iter().for_each(|r| match r {
response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer { .. } => {
primary_info = Some(r);
}
response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer { .. } => {
secondary_info = Some(r);
}
response::video_details::VideoResultsItem::None => {}
});
let (title, view_count, like_count, publish_date, publish_date_txt) = match primary_info {
Some(response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer {
title,
view_count,
video_actions,
date_text,
}) => {
let like_btn = some_or_bail!(
video_actions
.menu_renderer
.top_level_buttons
.into_iter()
.find(|button| {
button.toggle_button_renderer.default_icon.icon_type == IconType::Like
}),
Err(anyhow!("could not find like button"))
);
(
title,
util::parse_numeric(&view_count.video_view_count_renderer.view_count)?,
util::parse_numeric(&like_btn.toggle_button_renderer.accessibility_data)?,
timeago::parse_textual_date_or_warn(lang, &date_text, &mut warnings),
date_text,
)
}
_ => bail!("could not find primary_info"),
};
if publish_date.is_none() {
warnings.push(format!("could not parse date: {}", publish_date_txt));
}
let (owner, description, is_ccommons) = match secondary_info {
Some(response::video_details::VideoResultsItem::VideoSecondaryInfoRenderer {
owner,
description,
metadata_row_container,
}) => {
let is_ccommons = metadata_row_container
.map(|c| {
c.metadata_row_container_renderer.rows.iter().any(|cr| {
cr.metadata_row_renderer.contents.iter().any(|links| {
links.iter().any(|link| match link {
crate::serializer::text::TextLink::Web { text: _, url } => {
url == "https://www.youtube.com/t/creative_commons"
}
_ => false,
})
})
})
})
.unwrap_or_default();
(owner.video_owner_renderer, description, is_ccommons)
}
_ => bail!("could not find secondary_info"),
};
let (channel_id, channel_name) = match owner.title {
crate::serializer::text::TextLink::Browse {
text,
page_type,
browse_id,
} => match page_type {
crate::serializer::text::PageType::Channel => (browse_id, text),
_ => bail!("invalid channel link type"),
},
_ => bail!("invalid channel link"),
};
let mut recommended = map_recommendations(
self.contents
.two_column_watch_next_results
.secondary_results
.secondary_results
.results,
lang,
);
warnings.append(&mut recommended.warnings);
let mut engagement_panels = self.engagement_panels;
warnings.append(&mut engagement_panels.warnings);
let mut chapter_panel = None;
let mut comment_panel = None;
engagement_panels.c.into_iter().for_each(|panel| match panel.engagement_panel_section_list_renderer {
response::video_details::EngagementPanelRenderer::EngagementPanelMacroMarkersDescriptionChapters { content } => {
chapter_panel = Some(content);
},
response::video_details::EngagementPanelRenderer::EngagementPanelCommentsSection { header } => {
comment_panel = Some(header);
},
response::video_details::EngagementPanelRenderer::None => {},
});
let (top_comments_ctoken, latest_comments_ctoken) = match comment_panel {
Some(comments) => {
let mut items = comments
.engagement_panel_title_header_renderer
.menu
.sort_filter_sub_menu_renderer
.sub_menu_items;
let latest = items.try_swap_remove(1);
let top = items.try_swap_remove(0);
(
top.map(|c| c.service_endpoint.continuation_command.token),
latest.map(|c| c.service_endpoint.continuation_command.token),
)
}
None => (None, None),
};
Ok(MapResult {
c: VideoDetails {
id: id.to_owned(),
title: "".to_owned(),
description: "".to_owned(),
id: video_id,
title,
description,
channel: Channel {
id: channel_id,
name: channel_name,
avatar: owner.thumbnail.into(),
verification: owner.badges.into(),
subscriber_count: None,
subscriber_count_txt: owner.subscriber_count_text,
},
view_count,
like_count,
publish_date,
publish_date_txt,
is_ccommons,
recommended: recommended.c,
top_comments: Paginator {
ctoken: top_comments_ctoken,
..Default::default()
},
latest_comments: Paginator {
ctoken: latest_comments_ctoken,
..Default::default()
},
},
warnings: vec![],
warnings,
})
}
}
impl MapResponse<Paginator<RecommendedVideo>> for response::VideoRecommendations {
fn map_response(
self,
_id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<RecommendedVideo>>> {
let mut endpoints = self.on_response_received_endpoints;
let cont = some_or_bail!(
endpoints.try_swap_remove(0),
Err(anyhow!("no continuation endpoint"))
);
Ok(map_recommendations(
cont.append_continuation_items_action.continuation_items,
lang,
))
}
}
impl MapResponse<Paginator<Comment>> for response::VideoComments {
fn map_response(
self,
_id: &str,
lang: crate::model::Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Paginator<Comment>>> {
let mut warnings = self.on_response_received_endpoints.warnings;
let mut comments = Vec::new();
let mut comment_count = None;
let mut ctoken = None;
self.on_response_received_endpoints
.c
.into_iter()
.for_each(|citem| {
let mut items = citem.append_continuation_items_action.continuation_items;
warnings.append(&mut items.warnings);
items.c.into_iter().for_each(|item| match item {
response::video_details::CommentListItem::CommentThreadRenderer {
comment,
replies,
rendering_priority,
} => {
let mut res = map_comment(
comment.comment_renderer,
Some(replies),
rendering_priority,
lang,
);
comments.push(res.c);
warnings.append(&mut res.warnings)
}
response::video_details::CommentListItem::CommentRenderer { comment } => {
let mut res = map_comment(
comment,
None,
response::video_details::CommentPriority::RenderingPriorityUnknown,
lang,
);
comments.push(res.c);
warnings.append(&mut res.warnings)
}
response::video_details::CommentListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
ctoken = Some(continuation_endpoint.continuation_command.token);
}
response::video_details::CommentListItem::CommentsHeaderRenderer {
count_text,
} => {
comment_count = count_text.and_then(|txt| {
util::parse_numeric_or_warn::<u32>(&txt, &mut warnings)
});
}
});
});
Ok(MapResult {
c: Paginator {
count: comment_count,
items: comments,
ctoken,
},
warnings,
})
}
}
fn map_recommendations(
r: MapResult<Vec<response::VideoListItem<response::video_details::RecommendedVideo>>>,
lang: Language,
) -> MapResult<Paginator<RecommendedVideo>> {
let mut warnings = r.warnings;
let mut ctoken = None;
let items =
r.c.into_iter()
.filter_map(|item| match item {
response::VideoListItem::GridVideoRenderer { video } => {
match ChannelId::try_from(video.channel) {
Ok(channel) => Some(RecommendedVideo {
id: video.video_id,
title: video.title,
length: video.length_text.and_then(|txt| {
util::parse_video_length_or_warn(&txt, &mut warnings)
}),
thumbnail: video.thumbnail.into(),
channel: Channel {
id: channel.id,
name: channel.name,
avatar: video.channel_thumbnail.into(),
verification: video.owner_badges.into(),
subscriber_count: None,
subscriber_count_txt: None,
},
publish_date: video.published_time_text.as_ref().and_then(|txt| {
timeago::parse_timeago_or_warn(lang, txt, &mut warnings)
}),
publish_date_txt: video.published_time_text,
view_count: util::parse_numeric_or_warn(
&video.view_count_text,
&mut warnings,
),
is_live: video.badges.is_live(),
}),
Err(e) => {
warnings.push(e.to_string());
None
}
}
}
response::VideoListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
ctoken = Some(continuation_endpoint.continuation_command.token);
None
}
response::VideoListItem::None => None,
})
.collect::<Vec<_>>();
MapResult {
c: Paginator {
count: None,
items,
ctoken,
},
warnings,
}
}
fn map_comment(
c: response::video_details::CommentRenderer,
replies: Option<response::video_details::Replies>,
priority: response::video_details::CommentPriority,
lang: Language,
) -> MapResult<Comment> {
let mut warnings = Vec::new();
let mut reply_ctoken = None;
let replies = replies.map(|replies| {
replies
.comment_replies_renderer
.contents
.into_iter()
.filter_map(|item| match item {
response::video_details::CommentListItem::CommentRenderer { comment } => {
let mut res = map_comment(
comment,
None,
response::video_details::CommentPriority::default(),
lang,
);
warnings.append(&mut res.warnings);
Some(res.c)
}
response::video_details::CommentListItem::ContinuationItemRenderer {
continuation_endpoint,
} => {
reply_ctoken = Some(continuation_endpoint.continuation_command.token);
None
}
_ => None,
})
.collect::<Vec<_>>()
});
MapResult {
c: Comment {
id: c.comment_id,
text: c.content_text,
author: match (c.author_endpoint, c.author_text) {
(Some(aep), Some(name)) => Some(Channel {
id: aep.browse_endpoint.browse_id,
name,
avatar: c.author_thumbnail.into(),
verification: c
.author_comment_badge
.map(|b| b.author_comment_badge_renderer.icon.into())
.unwrap_or_default(),
subscriber_count: None,
subscriber_count_txt: None,
}),
_ => None,
},
publish_date: timeago::parse_timeago_or_warn(
lang,
&c.published_time_text,
&mut warnings,
),
publish_date_txt: c.published_time_text,
like_count: util::parse_numeric_or_warn(
&c.action_buttons
.comment_action_buttons_renderer
.like_button
.toggle_button_renderer
.accessibility_data,
&mut warnings,
),
reply_count: c.reply_count,
replies: replies
.map(|items| Paginator {
count: Some(c.reply_count),
items,
ctoken: reply_ctoken,
})
.unwrap_or_default(),
by_owner: c.author_is_channel_owner,
is_pinned: priority
== response::video_details::CommentPriority::RenderingPriorityPinnedComment,
is_hearted: c
.action_buttons
.comment_action_buttons_renderer
.creator_heart
.map(|h| h.creator_heart_renderer.is_hearted)
.unwrap_or_default(),
},
warnings,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::client::RustyPipe;
#[test_log::test(tokio::test)]
async fn get_video_details() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
dbg!(&details);
}
#[test_log::test(tokio::test)]
async fn get_video_recommendations() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let rec = rp
.query()
.video_recommendations(&details.recommended.ctoken.unwrap())
.await
.unwrap();
dbg!(&rec);
}
#[test_log::test(tokio::test)]
async fn get_video_comments() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("3pv_rHKnwAs").await.unwrap();
let rec = rp
.query()
.video_comments(&details.top_comments.ctoken.unwrap())
.await
.unwrap();
dbg!(&rec);
}
}

124
src/model/mod.rs
View file

@ -183,8 +183,8 @@ pub struct Playlist {
pub id: String,
pub name: String,
pub videos: Paginator<PlaylistVideo>,
pub n_videos: u32,
pub thumbnails: Vec<Thumbnail>,
pub video_count: u32,
pub thumbnail: Vec<Thumbnail>,
pub description: Option<String>,
pub channel: Option<ChannelId>,
pub last_update: Option<DateTime<Local>>,
@ -196,7 +196,7 @@ pub struct PlaylistVideo {
pub id: String,
pub title: String,
pub length: u32,
pub thumbnails: Vec<Thumbnail>,
pub thumbnail: Vec<Thumbnail>,
pub channel: ChannelId,
}
@ -212,24 +212,112 @@ pub struct ChannelId {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct VideoDetails {
/// Unique YouTube video ID
pub id: String,
/// Video title
pub title: String,
/// Video description
pub description: String,
/// Channel owning the video
pub channel: Channel,
/// Number of views
pub view_count: u64,
/// Number of likes
pub like_count: u32,
/// Video publish date. `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual video publish date (e.g. `Aug 2, 2013`, depends on language)
pub publish_date_txt: String,
/// Is the video published under the Creative Commons BY 3.0 license?
///
/// Information about the license:
///
/// https://www.youtube.com/t/creative_commons
///
/// https://creativecommons.org/licenses/by/3.0/
pub is_ccommons: bool,
/// Recommended videos
pub recommended: Paginator<RecommendedVideo>,
/// Paginator to fetch comments (most liked first)
pub top_comments: Paginator<Comment>,
/// Paginator to fetch comments (latest first)
pub latest_comments: Paginator<Comment>,
}
/*
@RECOMMENDATIONS
*/
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct RecommendedVideo {
/// Unique YouTube video ID
pub id: String,
/// Video title
pub title: String,
/// Video length in seconds.
///
/// Is `None` for livestreams.
pub length: Option<u32>,
/// Video thumbnail
pub thumbnail: Vec<Thumbnail>,
/// Channel owning the video
pub channel: Channel,
/// Video publish date. `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual video publish date (e.g. `11 months ago`, depends on language)
///
/// Is `None` for livestreams.
pub publish_date_txt: Option<String>,
/// View count
///
/// Is `None` if it could not be parsed
pub view_count: Option<u64>,
/// Is the video an active livestream?
pub is_live: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Channel {
/// Unique YouTube channel ID
pub id: String,
/// Channel name
pub name: String,
/// Channel avatar/profile picture
pub avatar: Vec<Thumbnail>,
/// Channel verification mark
pub verification: Verification,
/// Approximate number of subscribers
///
/// Info: This is only present in the `VideoDetails` response
pub subscriber_count: Option<u32>,
/// Textual subscriber count (e.g `1.41M subscribers`, depends on language)
pub subscriber_count_txt: Option<String>,
}
/*
@COMMENTS
*/
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Channel {
pub id: String,
pub name: String,
pub avatars: Vec<Thumbnail>,
pub verified: bool,
#[derive(Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum Verification {
#[default]
/// Unverified channel (default)
None,
/// Verified channel (✓ checkmark symbol)
Verified,
/// Verified music artist (♪ music note symbol)
Artist,
}
impl Verification {
pub fn verified(&self) -> bool {
self != &Self::None
}
}
// TODO: impl popularity comparison
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Comment {
/// Unique YouTube Comment-ID (e.g. `UgynScMrsqGSL8qvePl4AaABAg`)
pub id: String,
@ -239,16 +327,20 @@ pub struct Comment {
///
/// There may be comments with missing authors (possibly deleted users?).
pub author: Option<Channel>,
/// Number of upvotes
pub upvotes: u32,
/// Comment publish date. `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual comment publish date (e.g. `14 hours ago`), depends on language setting
pub publish_date_txt: String,
/// Number of comment likes
pub like_count: Option<u32>,
/// Number of replies
pub n_replies: u32,
pub reply_count: u32,
/// Paginator to fetch comment replies
pub replies: Paginator<Comment>,
/// Is the comment from the channel owner?
pub by_owner: bool,
/// Has the channel owner pinned the comment to the top?
pub pinned: bool,
/// Has the channel owner marked the comment with a ❤️ ?
pub hearted: bool,
pub is_pinned: bool,
/// Has the channel owner marked the comment with a ❤️ heart ?
pub is_hearted: bool,
}

5
src/model/paginator.rs
View file

@ -4,9 +4,8 @@ use serde::{Deserialize, Serialize};
/// The paginator is a wrapper around a list of items that are fetched
/// in pages from the YouTube API (e.g. playlist items,
/// video recommendations or comments).
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Paginator<T> {
/*
/// Total number of items if finite and known.
///
/// Note that this number may not be 100% accurate, as this is the
@ -18,7 +17,6 @@ pub struct Paginator<T> {
/// Don't use this number to check if all items were fetched or for
/// iterating over the items.
pub count: Option<u32>,
*/
/// Content of the paginator
pub items: Vec<T>,
/// The continuation token is passed to the YouTube API to fetch
@ -31,6 +29,7 @@ pub struct Paginator<T> {
impl<T> Default for Paginator<T> {
fn default() -> Self {
Self {
count: None,
items: Vec::new(),
ctoken: None,
}

31
src/serializer/text.rs
View file

@ -1,3 +1,6 @@
use std::convert::TryFrom;
use anyhow::anyhow;
use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DefaultOnError, DeserializeAs};
@ -240,12 +243,32 @@ impl<'de> DeserializeAs<'de, Vec<TextLink>> for TextLinks {
}
}
#[derive(Deserialize)]
pub struct AccessibilityText {
accessibility: AccessibilityData,
impl TryFrom<TextLink> for crate::model::ChannelId {
type Error = anyhow::Error;
fn try_from(value: TextLink) -> Result<Self, Self::Error> {
match value {
TextLink::Browse {
text,
page_type,
browse_id,
} => match page_type {
PageType::Channel => Ok(crate::model::ChannelId { id: browse_id, name: text }),
_ => Err(anyhow!("invalid channel link type")),
},
_ => Err(anyhow!("invalid channel link")),
}
}
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct AccessibilityText {
accessibility_data: AccessibilityData,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct AccessibilityData {
label: String,
}
@ -256,7 +279,7 @@ impl<'de> DeserializeAs<'de, String> for AccessibilityText {
D: Deserializer<'de>,
{
let text = AccessibilityText::deserialize(deserializer)?;
Ok(text.accessibility.label)
Ok(text.accessibility_data.label)
}
}

24
src/timeago.rs
View file

@ -148,6 +148,18 @@ pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option<DateTim
parse_timeago(lang, textual_date).map(|ta| ta.into())
}
pub(crate) fn parse_timeago_or_warn(
lang: Language,
textual_date: &str,
warnings: &mut Vec<String>,
) -> Option<DateTime<Local>> {
let res = parse_timeago_to_dt(lang, textual_date);
if res.is_none() {
warnings.push(format!("could not parse timeago `{}`", textual_date));
}
res
}
pub fn parse_textual_date(lang: Language, textual_date: &str) -> Option<ParsedDate> {
let entry = dictionary::entry(lang);
let filtered_str = filter_str(textual_date);
@ -198,6 +210,18 @@ pub fn parse_textual_date_to_dt(lang: Language, textual_date: &str) -> Option<Da
parse_textual_date(lang, textual_date).map(|ta| ta.into())
}
pub(crate) fn parse_textual_date_or_warn(
lang: Language,
textual_date: &str,
warnings: &mut Vec<String>,
) -> Option<DateTime<Local>> {
let res = parse_textual_date_to_dt(lang, textual_date);
if res.is_none() {
warnings.push(format!("could not parse timeago `{}`", textual_date));
}
res
}
#[cfg(test)]
mod tests {
use std::{collections::BTreeMap, fs::File, io::BufReader, path::Path};

55
src/util.rs
View file

@ -2,6 +2,7 @@ use std::{collections::BTreeMap, str::FromStr};
use anyhow::Result;
use fancy_regex::Regex;
use once_cell::sync::Lazy;
use rand::Rng;
use url::Url;
@ -89,6 +90,48 @@ where
numbers
}
/// Parse textual video length (e.g. `0:49`, `2:02` or `1:48:18`)
/// and return the duration in seconds.
pub fn parse_video_length(text: &str) -> Option<u32> {
static VIDEO_LENGTH_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"(?:(\d+):)?(\d{1,2}):(\d{2})"#).unwrap());
VIDEO_LENGTH_REGEX.captures(text).ok().flatten().map(|cap| {
let hrs = cap
.get(1)
.and_then(|x| x.as_str().parse::<u32>().ok())
.unwrap_or_default();
let min = cap
.get(2)
.and_then(|x| x.as_str().parse::<u32>().ok())
.unwrap_or_default();
let sec = cap
.get(3)
.and_then(|x| x.as_str().parse::<u32>().ok())
.unwrap_or_default();
hrs * 3600 + min * 60 + sec
})
}
pub fn parse_numeric_or_warn<F>(string: &str, warnings: &mut Vec<String>) -> Option<F>
where
F: FromStr,
{
let res = parse_numeric::<F>(string);
if res.is_err() {
warnings.push(format!("could not parse number `{}`", string));
}
res.ok()
}
pub fn parse_video_length_or_warn(text: &str, warnings: &mut Vec<String>) -> Option<u32> {
let res = parse_video_length(text);
if res.is_none() {
warnings.push(format!("could not parse video length `{}`", text));
}
res
}
pub fn retry_delay(
n_past_retries: u32,
min_retry_interval: u32,
@ -167,6 +210,18 @@ mod tests {
assert_eq!(n, expect);
}
#[rstest]
#[case("0:49", Some(49))]
#[case("bla 2:02 h3llo w0rld", Some(122))]
#[case("18:22", Some(1102))]
#[case("1:48:18", Some(6498))]
#[case("102:12:39", Some(367959))]
#[case("42", None)]
fn t_parse_video_length(#[case] text: &str, #[case] expect: Option<u32>) {
let n = parse_video_length(text);
assert_eq!(n, expect);
}
#[rstest]
#[case(0, 800, 1500)]
#[case(1, 2400, 4500)]

12846
testfiles/video_details/comments_top.json Normal file
View file

File diff suppressed because it is too large Load diff

12728
testfiles/video_details/comments_top_with_count.json Normal file
View file

File diff suppressed because it is too large Load diff