Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add is_live to video details

Browse source
This commit is contained in:
ThetaDev 2022-09-20 21:22:18 +02:00
parent 8c1e7bf6ac
commit 584d6aa3f5
9 changed files with 28819 additions and 7735 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Justfile
View file

@ -1,3 +1,3 @@
report2yaml:
yq e -Pi rustypipe_reports/*.json
for f in rustypipe_reports/*.json; do mv $f rustypipe_reports/`basename $f .json`.yaml; done;
mkdir -p rustypipe_reports/conv
for f in rustypipe_reports/*.json; do yq '.http_request.resp_body' $f | yq -o json -P > rustypipe_reports/conv/`basename $f .json`_body.json; yq e -Pi $f; mv $f rustypipe_reports/conv/`basename $f .json`.yaml; done;

33
codegen/src/download_testfiles.rs
View file

@ -123,10 +123,12 @@ async fn playlist(testfiles: &Path) {
async fn video_details(testfiles: &Path) {
for (name, id) in [
("music", "MZOgTu2dMTg"),
("music", "XuM2onMGvTI"),
("mv", "ZeerrnuLi5E"),
("ccommons", "0rb9CfOvojk"),
("chapters", "nFDBxBUfE74"),
("agegate", "XuM2onMGvTI"),
("live", "86YLFOog4GM"),
] {
let mut json_path = testfiles.to_path_buf();
json_path.push("video_details");
@ -156,33 +158,4 @@ async fn comments_top(testfiles: &Path) {
.video_comments(&details.top_comments.ctoken.unwrap())
.await
.unwrap();
// rp.query().video_comments("Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D").await.unwrap();
// Desktop 1
// top: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// latest: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczABeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// shows count
// Desktop 2
// top: Eg0SC0lITnpPSGk4c0pzGAYyVSIuIgtJSE56T0hpOHNKczAAeAKqAhpVZ3lVZG5WQnBNR09tMnVMR3o1NEFhQUJBZzABQiFlbmdhZ2VtZW50LXBhbmVsLWNvbW1lbnRzLXNlY3Rpb24%3D
// shows no count
// latest: Eg0SC0lITnpPSGk4c0pzGAYyOCIRIgtJSE56T0hpOHNKczABeAIwAUIhZW5nYWdlbWVudC1wYW5lbC1jb21tZW50cy1zZWN0aW9u
// shows no count
// cont: Eg0SC0lITnpPSGk4c0pzGAYyJSIRIgtJSE56T0hpOHNKczAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D
// shows count
}
#[tokio::test]
async fn test() {
let id = "IHNzOHi8sJs";
let rp = RustyPipe::new();
let details = rp.query().video_details(id).await.unwrap();
let ctoken_top = details.top_comments.ctoken;
let ctoken_latest = details.latest_comments.ctoken;
dbg!(ctoken_top);
dbg!(ctoken_latest);
}

3
notes/video_ids.txt
View file

@ -20,6 +20,9 @@ Album with unknown artists: https://music.youtube.com/playlist?list=OLAK5uy_mEX9
Comment by artist: 3pv_rHKnwAs
Comments disabled: XuM2onMGvTI
Likes hidden:
# Playlists
962 Songs: PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI
97 Songs, YTM: RDCLAK5uy_kFQXdnqMaQCVx2wpUM4ZfbsGCDibZtkJk

42
src/client/response/video_details.rs
View file

@ -47,7 +47,9 @@ pub struct TwoColumnWatchNextResults {
/// Metadata about the video
pub results: VideoResultsWrap,
/// Video recommendations
pub secondary_results: RecommendationResultsWrap,
///
/// Can be `None` for age-restricted videos
pub secondary_results: Option<RecommendationResultsWrap>,
}
/// Metadata about the video
@ -85,6 +87,7 @@ pub enum VideoResultsItem {
#[serde(rename_all = "camelCase")]
VideoSecondaryInfoRenderer {
owner: VideoOwner,
#[serde(default)]
#[serde_as(as = "Text")]
description: String,
/// Additional metadata (e.g. Creative Commons License)
@ -114,6 +117,8 @@ pub struct ViewCountRenderer {
/// View count (`232,975,196 views`)
#[serde_as(as = "Text")]
pub view_count: String,
#[serde(default)]
pub is_live: bool,
}
/// Like/Dislike buttons
@ -129,7 +134,23 @@ pub struct VideoActions {
#[serde(rename_all = "camelCase")]
pub struct VideoActionsMenu {
#[serde_as(as = "VecSkipError<_>")]
pub top_level_buttons: Vec<ToggleButtonWrap>,
pub top_level_buttons: Vec<TopLevelButton>,
}
/// The different TopLevelButtons
///
/// YouTube seems to be A/B testing the SegmentedLikeDislikeButtonRenderer
///
/// See: https://github.com/TeamNewPipe/NewPipeExtractor/pull/926
#[serde_as]
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum TopLevelButton {
ToggleButtonRenderer(ToggleButton),
#[serde(rename_all = "camelCase")]
SegmentedLikeDislikeButtonRenderer {
like_button: ToggleButtonWrap,
},
}
/// Like/Dislike button
@ -147,6 +168,8 @@ pub struct ToggleButton {
/// Icon type: `LIKE` / `DISLIKE`
pub default_icon: Icon,
/// Number of likes (`like this video along with 4,010,156 other people`)
///
/// Contains no digits (e.g. `I like this`) if likes are hidden by the creator.
#[serde_as(as = "AccessibilityText")]
pub accessibility_data: String,
}
@ -257,8 +280,9 @@ pub struct RecommendationResultsWrap {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RecommendationResults {
#[serde_as(as = "VecLogError<_>")]
pub results: MapResult<Vec<VideoListItem<RecommendedVideo>>>,
/// Can be `None` for age-restricted videos
#[serde_as(as = "Option<VecLogError<_>>")]
pub results: Option<MapResult<Vec<VideoListItem<RecommendedVideo>>>>,
}
/// Video recommendation item
@ -313,7 +337,7 @@ pub enum EngagementPanelRenderer {
/// Ignored items:
/// - `engagement-panel-ads`
/// - `engagement-panel-structured-description`
/// (Desctiption already included in `VideoSecondaryInfoRenderer`)
/// (Description already included in `VideoSecondaryInfoRenderer`)
/// - `engagement-panel-searchable-transcript`
/// (basically video subtitles in a different format)
#[serde(other, deserialize_with = "ignore_any")]
@ -378,11 +402,13 @@ pub struct CommentItemSectionHeader {
#[derive(Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CommentItemSectionHeaderRenderer {
/// Average comment count (e.g. `81`, `2.2K`, `705K`)
/// Approximate comment count (e.g. `81`, `2.2K`, `705K`)
///
/// The accurate count is included in the first comment response.
#[serde_as(as = "Text")]
pub contextual_info: String,
///
/// Is `None` if there are no comments.
#[serde_as(as = "Option<Text>")]
pub contextual_info: Option<String>,
pub menu: CommentItemSectionHeaderMenu,
}

89
src/client/video_details.rs
View file

@ -138,33 +138,41 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
response::video_details::VideoResultsItem::None => {}
});
let (title, view_count, like_count, publish_date, publish_date_txt) = match primary_info {
Some(response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer {
title,
view_count,
video_actions,
date_text,
}) => {
let like_btn = some_or_bail!(
video_actions
.menu_renderer
.top_level_buttons
.into_iter()
.find(|button| {
button.toggle_button_renderer.default_icon.icon_type == IconType::Like
}),
Err(anyhow!("could not find like button"))
);
(
let (title, view_count, like_count, publish_date, publish_date_txt, is_live) =
match primary_info {
Some(response::video_details::VideoResultsItem::VideoPrimaryInfoRenderer {
title,
util::parse_numeric(&view_count.video_view_count_renderer.view_count)?,
util::parse_numeric(&like_btn.toggle_button_renderer.accessibility_data)?,
timeago::parse_textual_date_or_warn(lang, &date_text, &mut warnings),
view_count,
video_actions,
date_text,
)
}
_ => bail!("could not find primary_info"),
};
}) => {
let like_btn = video_actions
.menu_renderer
.top_level_buttons
.into_iter()
.find_map(|button| {
let btn = match button {
response::video_details::TopLevelButton::ToggleButtonRenderer(btn) => btn,
response::video_details::TopLevelButton::SegmentedLikeDislikeButtonRenderer { like_button } => like_button.toggle_button_renderer,
};
match btn.default_icon.icon_type {
IconType::Like => Some(btn),
_ => None
}
});
(
title,
util::parse_numeric(&view_count.video_view_count_renderer.view_count)?,
// accessibility_data contains no digits if the like count is hidden,
// so we ignore parse errors here for now
like_btn.and_then(|btn| util::parse_numeric(&btn.accessibility_data).ok()),
timeago::parse_textual_date_or_warn(lang, &date_text, &mut warnings),
date_text,
view_count.video_view_count_renderer.is_live,
)
}
_ => bail!("could not find primary_info"),
};
/*
TODO: use large number parser for this
@ -220,15 +228,19 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
_ => bail!("invalid channel link"),
};
let mut recommended = map_recommendations(
self.contents
.two_column_watch_next_results
.secondary_results
.secondary_results
.results,
lang,
);
warnings.append(&mut recommended.warnings);
let recommended = self
.contents
.two_column_watch_next_results
.secondary_results
.map(|sr| {
sr.secondary_results.results.map(|r| {
let mut res = map_recommendations(r, lang);
warnings.append(&mut res.warnings);
res.c
})
})
.flatten()
.unwrap_or_default();
let mut engagement_panels = self.engagement_panels;
warnings.append(&mut engagement_panels.warnings);
@ -273,8 +285,9 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
like_count,
publish_date,
publish_date_txt,
is_live,
is_ccommons,
recommended: recommended.c,
recommended,
top_comments: Paginator {
count: None,
items: Vec::new(),
@ -521,9 +534,9 @@ fn map_comment(
})
.unwrap_or_default(),
by_owner: c.author_is_channel_owner,
is_pinned: priority
pinned: priority
== response::video_details::CommentPriority::RenderingPriorityPinnedComment,
is_hearted: c
hearted: c
.action_buttons
.comment_action_buttons_renderer
.creator_heart
@ -541,7 +554,7 @@ mod tests {
#[test_log::test(tokio::test)]
async fn get_video_details() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let details = rp.query().video_details("HRKu0cvrr_o").await.unwrap();
dbg!(&details);
}

30
src/model/mod.rs
View file

@ -218,16 +218,22 @@ pub struct VideoDetails {
pub title: String,
/// Video description
pub description: String,
/// Channel owning the video
/// Channel of the video
pub channel: Channel,
/// Number of views
pub view_count: u64,
/// Number of likes
pub like_count: u32,
/// Video publish date. `None` if the date could not be parsed.
///
/// `None` if the like count was hidden by the creator.
pub like_count: Option<u32>,
/// Video publishing date. Start date in case of a livestream.
///
/// `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual video publish date (e.g. `Aug 2, 2013`, depends on language)
/// Textual video publishing date (e.g. `Aug 2, 2013`, depends on language)
pub publish_date_txt: String,
/// Is the video a livestream?
pub is_live: bool,
/// Is the video published under the Creative Commons BY 3.0 license?
///
/// Information about the license:
@ -237,6 +243,8 @@ pub struct VideoDetails {
/// https://creativecommons.org/licenses/by/3.0/
pub is_ccommons: bool,
/// Recommended videos
///
/// Note: Recommendations are not available for age-restricted videos
pub recommended: Paginator<RecommendedVideo>,
/// Paginator to fetch comments (most liked first)
pub top_comments: Paginator<Comment>,
@ -260,9 +268,11 @@ pub struct RecommendedVideo {
pub length: Option<u32>,
/// Video thumbnail
pub thumbnail: Vec<Thumbnail>,
/// Channel owning the video
/// Channel of the video
pub channel: Channel,
/// Video publish date. `None` if the date could not be parsed.
/// Video publishing date.
///
/// `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual video publish date (e.g. `11 months ago`, depends on language)
///
@ -327,7 +337,9 @@ pub struct Comment {
///
/// There may be comments with missing authors (possibly deleted users?).
pub author: Option<Channel>,
/// Comment publish date. `None` if the date could not be parsed.
/// Comment publishing date.
///
/// `None` if the date could not be parsed.
pub publish_date: Option<DateTime<Local>>,
/// Textual comment publish date (e.g. `14 hours ago`), depends on language setting
pub publish_date_txt: String,
@ -340,7 +352,7 @@ pub struct Comment {
/// Is the comment from the channel owner?
pub by_owner: bool,
/// Has the channel owner pinned the comment to the top?
pub is_pinned: bool,
pub pinned: bool,
/// Has the channel owner marked the comment with a ❤️ heart ?
pub is_hearted: bool,
pub hearted: bool,
}

10215
testfiles/video_details/video_details_agegate.json Normal file
View file

File diff suppressed because it is too large Load diff

13192
testfiles/video_details/video_details_live.json Normal file
View file

File diff suppressed because it is too large Load diff

12946
testfiles/video_details/video_details_music.json
View file

File diff suppressed because it is too large Load diff