From ecb84e32e1639e5ff6585e06393cd64fc22fa1d2 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 10 Oct 2022 01:09:13 +0200 Subject: [PATCH] feat: add search --- src/client/channel.rs | 18 +- src/client/channel_rss.rs | 2 +- src/client/mod.rs | 37 ++-- src/client/pagination.rs | 46 ++++- src/client/player.rs | 8 +- src/client/playlist.rs | 4 +- src/client/response/channel_rss.rs | 2 +- src/client/response/mod.rs | 11 +- src/client/response/search.rs | 215 ++++++++++++++++++++++ src/client/response/video_details.rs | 10 +- src/client/search.rs | 265 +++++++++++++++++++++++++++ src/client/video_details.rs | 14 +- src/download.rs | 36 ++-- src/error.rs | 2 +- src/model/mod.rs | 101 +++++++++- src/model/paginator.rs | 4 +- src/report.rs | 7 +- src/serializer/text.rs | 27 ++- src/{util.rs => util/mod.rs} | 7 +- 19 files changed, 710 insertions(+), 106 deletions(-) create mode 100644 src/client/response/search.rs create mode 100644 src/client/search.rs rename src/{util.rs => util/mod.rs} (98%) diff --git a/src/client/channel.rs b/src/client/channel.rs index 4ea2c4f..7b74532 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -19,9 +19,9 @@ use super::{ #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct QChannel { +struct QChannel<'a> { context: YTContext, - browse_id: String, + browse_id: &'a str, params: Params, } @@ -56,7 +56,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, - browse_id: channel_id.to_owned(), + browse_id: channel_id, params: match order { ChannelOrder::Latest => Params::VideosLatest, ChannelOrder::Oldest => Params::VideosOldest, @@ -81,7 +81,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, - continuation: ctoken.to_owned(), + continuation: ctoken, }; self.execute_request::( @@ -101,7 +101,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, - browse_id: channel_id.to_owned(), + browse_id: channel_id, params: Params::Playlists, }; @@ -122,7 +122,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, - continuation: ctoken.to_owned(), + continuation: ctoken, }; self.execute_request::( @@ -139,7 +139,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QChannel { context, - browse_id: channel_id.to_owned(), + browse_id: channel_id, params: Params::Info, }; @@ -180,7 +180,7 @@ impl MapResponse>> for response::Channel { id, lang, )?, - warnings: warnings, + warnings, }) } } @@ -211,7 +211,7 @@ impl MapResponse>> for response::Channel { id, lang, )?, - warnings: warnings, + warnings, }) } } diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index 33c6e6a..44fa26b 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -31,7 +31,7 @@ impl RustyPipeQuery { msgs: Vec::new(), deobf_data: None, http_request: crate::report::HTTPRequest { - url: url, + url, method: "GET".to_owned(), req_header: BTreeMap::new(), req_body: String::new(), diff --git a/src/client/mod.rs b/src/client/mod.rs index 4de9d87..e93ade5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -5,6 +5,7 @@ mod pagination; mod player; mod playlist; mod response; +mod search; mod video_details; #[cfg(feature = "rss")] @@ -124,9 +125,9 @@ struct ThirdParty { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct QContinuation { +struct QContinuation<'a> { context: YTContext, - continuation: String, + continuation: &'a str, } const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"; @@ -514,11 +515,11 @@ impl RustyPipe { ) .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from( - ExtractionError::InvalidData( + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or_else(|| { + Error::from(ExtractionError::InvalidData( "Could not find desktop client version in sw.js".into(), - ), - )) + )) + }) }; let from_html = async { @@ -532,11 +533,11 @@ impl RustyPipe { ) .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from( - ExtractionError::InvalidData( + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or_else(|| { + Error::from(ExtractionError::InvalidData( "Could not find desktop client version in sw.js".into(), - ), - )) + )) + }) }; match from_swjs.await { @@ -561,9 +562,11 @@ impl RustyPipe { ) .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or(Error::from( - ExtractionError::InvalidData("Could not find music client version in sw.js".into()), - )) + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &swjs, 1).ok_or_else(|| { + Error::from(ExtractionError::InvalidData( + "Could not find music client version in sw.js".into(), + )) + }) }; let from_html = async { @@ -577,11 +580,11 @@ impl RustyPipe { ) .await?; - util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or(Error::from( - ExtractionError::InvalidData( + util::get_cg_from_regexes(CLIENT_VERSION_REGEXES.iter(), &html, 1).ok_or_else(|| { + Error::from(ExtractionError::InvalidData( "Could not find music client version on html page".into(), - ), - )) + )) + }) }; match from_swjs.await { diff --git a/src/client/pagination.rs b/src/client/pagination.rs index ae32195..3ebd019 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,7 +1,7 @@ use crate::error::Result; use crate::model::{ - ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, + ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, SearchItem, }; use super::RustyPipeQuery; @@ -225,3 +225,47 @@ impl Paginator { Ok(()) } } + +impl Paginator { + pub async fn next(&self, query: RustyPipeQuery) -> Result> { + Ok(match &self.ctoken { + Some(ctoken) => Some(query.search_continuation(ctoken).await?), + None => None, + }) + } + + pub async fn extend(&mut self, query: RustyPipeQuery) -> Result { + match self.next(query).await { + Ok(Some(paginator)) => { + let mut items = paginator.items; + self.items.append(&mut items); + self.ctoken = paginator.ctoken; + Ok(true) + } + Ok(None) => Ok(false), + Err(e) => Err(e), + } + } + + pub async fn extend_pages(&mut self, query: RustyPipeQuery, n_pages: usize) -> Result<()> { + for _ in 0..n_pages { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } + + pub async fn extend_limit(&mut self, query: RustyPipeQuery, n_items: usize) -> Result<()> { + while self.items.len() < n_items { + match self.extend(query.clone()).await { + Ok(false) => break, + Err(e) => return Err(e), + _ => {} + } + } + Ok(()) + } +} diff --git a/src/client/player.rs b/src/client/player.rs index 457bdd2..3a0a584 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -26,7 +26,7 @@ use super::{ #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] -struct QPlayer { +struct QPlayer<'a> { context: YTContext, /// Website playback context #[serde(skip_serializing_if = "Option::is_none")] @@ -35,7 +35,7 @@ struct QPlayer { #[serde(skip_serializing_if = "Option::is_none")] cpn: Option, /// YouTube video ID - video_id: String, + video_id: &'a str, /// Set to true to allow extraction of streams with sensitive content content_check_ok: bool, /// Probably refers to allowing sensitive content, too @@ -82,7 +82,7 @@ impl RustyPipeQuery { }, }), cpn: None, - video_id: video_id.to_owned(), + video_id, content_check_ok: true, racy_check_ok: true, } @@ -91,7 +91,7 @@ impl RustyPipeQuery { context, playback_context: None, cpn: Some(util::generate_content_playback_nonce()), - video_id: video_id.to_owned(), + video_id, content_check_ok: true, racy_check_ok: true, } diff --git a/src/client/playlist.rs b/src/client/playlist.rs index c9e6095..5dea1b1 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -46,7 +46,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, - continuation: ctoken.to_owned(), + continuation: ctoken, }; self.execute_request::( @@ -143,7 +143,7 @@ impl MapResponse for response::Playlist { Err(ExtractionError::InvalidData("no video count".into())) ) } - None => videos.len() as u32, + None => videos.len() as u64, }; let playlist_id = self.header.playlist_header_renderer.playlist_id; diff --git a/src/client/response/channel_rss.rs b/src/client/response/channel_rss.rs index e5eff72..205f103 100644 --- a/src/client/response/channel_rss.rs +++ b/src/client/response/channel_rss.rs @@ -48,7 +48,7 @@ pub struct Community { #[derive(Debug, Deserialize)] pub struct Rating { - pub count: u32, + pub count: u64, } #[derive(Debug, Deserialize)] diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 8ee2c6a..e84fb96 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -2,6 +2,7 @@ pub mod channel; pub mod player; pub mod playlist; pub mod playlist_music; +pub mod search; pub mod video_details; pub use channel::Channel; @@ -10,6 +11,8 @@ pub use player::Player; pub use playlist::Playlist; pub use playlist::PlaylistCont; pub use playlist_music::PlaylistMusic; +pub use search::Search; +pub use search::SearchCont; pub use video_details::VideoComments; pub use video_details::VideoDetails; pub use video_details::VideoRecommendations; @@ -67,10 +70,14 @@ pub struct Thumbnail { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub enum VideoListItem { + /// Video on channel page GridVideoRenderer(GridVideoRenderer), + /// Video in recommendations CompactVideoRenderer(CompactVideoRenderer), + /// Video in playlist PlaylistVideoRenderer(PlaylistVideoRenderer), + /// Playlist on channel page GridPlaylistRenderer(GridPlaylistRenderer), /// Continauation items are located at the end of a list @@ -266,7 +273,7 @@ pub struct TimeOverlay { pub struct TimeOverlayRenderer { /// `29:54` /// - /// Is `LIVE` in case of a livestream + /// Is `LIVE` in case of a livestream and `SHORTS` in case of a short video #[serde_as(as = "Text")] pub text: String, #[serde(default)] @@ -274,7 +281,7 @@ pub struct TimeOverlayRenderer { pub style: TimeOverlayStyle, } -#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash)] +#[derive(Default, Clone, Copy, Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum TimeOverlayStyle { #[default] diff --git a/src/client/response/search.rs b/src/client/response/search.rs new file mode 100644 index 0000000..fe7509b --- /dev/null +++ b/src/client/response/search.rs @@ -0,0 +1,215 @@ +use serde::Deserialize; +use serde_with::json::JsonString; +use serde_with::{serde_as, VecSkipError}; + +use crate::serializer::ignore_any; +use crate::serializer::{ + text::{Text, TextComponent}, + MapResult, VecLogError, +}; + +use super::{ChannelBadge, ContentsRenderer, ContinuationEndpoint, Thumbnails, TimeOverlay}; + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Search { + #[serde_as(as = "Option")] + pub estimated_results: Option, + pub contents: Contents, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchCont { + #[serde_as(as = "Option")] + pub estimated_results: Option, + pub on_response_received_commands: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchContCommand { + pub append_continuation_items_action: SearchContAction, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchContAction { + pub continuation_items: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Contents { + pub two_column_search_results_renderer: TwoColumnSearchResultsRenderer, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TwoColumnSearchResultsRenderer { + pub primary_contents: PrimaryContents, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrimaryContents { + pub section_list_renderer: ContentsRenderer, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SectionListItem { + #[serde(rename_all = "camelCase")] + ItemSectionRenderer { + #[serde_as(as = "VecLogError<_>")] + contents: MapResult>, + }, + /// Continuation token to fetch more search results + #[serde(rename_all = "camelCase")] + ContinuationItemRenderer { + continuation_endpoint: ContinuationEndpoint, + }, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SearchItem { + /// Video in search results + VideoRenderer(VideoRenderer), + /// Playlist in search results + PlaylistRenderer(PlaylistRenderer), + /// Channel displayed in search results + ChannelRenderer(ChannelRenderer), + + /// Corrected search query + #[serde(rename_all = "camelCase")] + ShowingResultsForRenderer { + #[serde_as(as = "Text")] + corrected_query: String, + }, + /// No search result item (e.g. ad) or unimplemented item + /// + /// Unimplemented: + /// - shelfRenderer (e.g. Latest from channel, For you) + #[serde(other, deserialize_with = "ignore_any")] + None, +} + +/// Video displayed in search results +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VideoRenderer { + pub video_id: String, + pub thumbnail: Thumbnails, + #[serde_as(as = "Text")] + pub title: String, + #[serde(rename = "shortBylineText")] + pub channel: TextComponent, + pub channel_thumbnail_supported_renderers: ChannelThumbnailSupportedRenderers, + #[serde_as(as = "Option")] + pub published_time_text: Option, + #[serde_as(as = "Option")] + pub length_text: Option, + /// Contains `No views` if the view count is zero + #[serde_as(as = "Option")] + pub view_count_text: Option, + /// Channel verification badge + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub owner_badges: Vec, + /// Contains Short/Live tag + #[serde_as(as = "VecSkipError<_>")] + pub thumbnail_overlays: Vec, + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub detailed_metadata_snippets: Vec, +} + +/// Playlist displayed in search results +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlaylistRenderer { + pub playlist_id: String, + #[serde_as(as = "Text")] + pub title: String, + /// The first item of this list contains the playlist thumbnail, + /// subsequent items contain very small thumbnails of the next playlist videos + pub thumbnails: Vec, + #[serde_as(as = "JsonString")] + pub video_count: u64, + #[serde(rename = "shortBylineText")] + pub channel: TextComponent, + /// Channel verification badge + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub owner_badges: Vec, + /// First 2 videos + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub videos: Vec, +} + +/// Channel displayed in search results +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChannelRenderer { + pub channel_id: String, + #[serde_as(as = "Text")] + pub title: String, + pub thumbnail: Thumbnails, + /// Abbreviated channel description + #[serde_as(as = "Text")] + pub description_snippet: String, + #[serde_as(as = "Text")] + pub video_count_text: String, + #[serde_as(as = "Option")] + pub subscriber_count_text: Option, + /// Channel verification badge + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub owner_badges: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChannelThumbnailSupportedRenderers { + pub channel_thumbnail_with_link_renderer: ChannelThumbnailWithLinkRenderer, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChannelThumbnailWithLinkRenderer { + pub thumbnail: Thumbnails, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChildVideoRendererWrap { + pub child_video_renderer: ChildVideoRenderer, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChildVideoRenderer { + pub video_id: String, + #[serde_as(as = "Text")] + pub title: String, + #[serde_as(as = "Option")] + pub length_text: Option, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DetailedMetadataSnippet { + #[serde_as(as = "Text")] + pub snippet_text: String, +} diff --git a/src/client/response/video_details.rs b/src/client/response/video_details.rs index 4b9b6f1..f453c56 100644 --- a/src/client/response/video_details.rs +++ b/src/client/response/video_details.rs @@ -4,12 +4,10 @@ use serde::Deserialize; use serde_with::serde_as; use serde_with::{DefaultOnError, VecSkipError}; -use crate::serializer::text::TextComponents; -use crate::serializer::MapResult; use crate::serializer::{ ignore_any, - text::{AccessibilityText, AttributedText, Text}, - VecLogError, + text::{AccessibilityText, AttributedText, Text, TextComponents}, + MapResult, VecLogError, }; use super::{ @@ -29,8 +27,8 @@ pub struct VideoDetails { pub contents: Contents, /// Video ID pub current_video_endpoint: CurrentVideoEndpoint, - #[serde_as(as = "VecLogError<_>")] /// Video chapters + comment section + #[serde_as(as = "VecLogError<_>")] pub engagement_panels: MapResult>, } @@ -546,7 +544,7 @@ pub struct CommentRenderer { // pub vote_count: Option, pub author_comment_badge: Option, #[serde(default)] - pub reply_count: u32, + pub reply_count: u64, /// Buttons for comment interaction (Like/Dislike/Reply) pub action_buttons: CommentActionButtons, } diff --git a/src/client/search.rs b/src/client/search.rs new file mode 100644 index 0000000..2846037 --- /dev/null +++ b/src/client/search.rs @@ -0,0 +1,265 @@ +use serde::Serialize; + +use crate::{ + deobfuscate::Deobfuscator, + error::{Error, ExtractionError}, + model::{ + ChannelId, ChannelTag, Language, Paginator, SearchChannel, SearchItem, SearchPlaylist, + SearchPlaylistVideo, SearchResult, SearchVideo, + }, + timeago, + util::{self, TryRemove}, +}; + +use super::{ + response::{self, IsLive, IsShort}, + ClientType, MapResponse, MapResult, QContinuation, RustyPipeQuery, YTContext, +}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct QSearch<'a> { + context: YTContext, + query: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + params: Option, +} + +impl RustyPipeQuery { + pub async fn search(self, query: &str) -> Result { + let context = self.get_context(ClientType::Desktop, true).await; + let request_body = QSearch { + context, + query, + params: None, + }; + + self.execute_request::( + ClientType::Desktop, + "search", + query, + "search", + &request_body, + ) + .await + } + + pub async fn search_continuation(self, ctoken: &str) -> Result, Error> { + let context = self.get_context(ClientType::Desktop, true).await; + let request_body = QContinuation { + context, + continuation: ctoken, + }; + + self.execute_request::( + ClientType::Desktop, + "search", + ctoken, + "search", + &request_body, + ) + .await + } +} + +impl MapResponse for response::Search { + fn map_response( + self, + _id: &str, + lang: Language, + _deobf: Option<&Deobfuscator>, + ) -> Result, ExtractionError> { + let section_list_items = self + .contents + .two_column_search_results_renderer + .primary_contents + .section_list_renderer + .contents; + + let (items, ctoken) = map_section_list_items(section_list_items)?; + + let mut warnings = items.warnings; + let (mut mapped, corrected_query) = map_search_items(items.c, lang); + warnings.append(&mut mapped.warnings); + + Ok(MapResult { + c: SearchResult { + items: Paginator::new(self.estimated_results, mapped.c, ctoken), + corrected_query, + }, + warnings, + }) + } +} + +impl MapResponse> for response::SearchCont { + fn map_response( + self, + _id: &str, + lang: Language, + _deobf: Option<&Deobfuscator>, + ) -> Result>, ExtractionError> { + let mut commands = self.on_response_received_commands; + let cont_command = some_or_bail!( + commands.try_swap_remove(0), + Err(ExtractionError::InvalidData( + "no item section renderer".into() + )) + ); + + let (items, ctoken) = map_section_list_items( + cont_command + .append_continuation_items_action + .continuation_items, + )?; + + let mut warnings = items.warnings; + let (mut mapped, _) = map_search_items(items.c, lang); + warnings.append(&mut mapped.warnings); + + Ok(MapResult { + c: Paginator::new(self.estimated_results, mapped.c, ctoken), + warnings, + }) + } +} + +fn map_section_list_items( + section_list_items: Vec, +) -> Result<(MapResult>, Option), ExtractionError> { + let mut items = None; + let mut ctoken = None; + section_list_items.into_iter().for_each(|item| match item { + response::search::SectionListItem::ItemSectionRenderer { contents } => { + items = Some(contents); + } + response::search::SectionListItem::ContinuationItemRenderer { + continuation_endpoint, + } => { + ctoken = Some(continuation_endpoint.continuation_command.token); + } + }); + + let items = some_or_bail!( + items, + Err(ExtractionError::InvalidData( + "no item section renderer".into() + )) + ); + + Ok((items, ctoken)) +} + +fn map_search_items( + items: Vec, + lang: Language, +) -> (MapResult>, Option) { + let mut warnings = Vec::new(); + + let mut c_query = None; + let mapped_items = items + .into_iter() + .filter_map(|item| match item { + response::search::SearchItem::VideoRenderer(mut video) => { + match ChannelId::try_from(video.channel) { + Ok(channel) => Some(SearchItem::Video(SearchVideo { + id: video.video_id, + title: video.title, + length: video + .length_text + .and_then(|txt| util::parse_video_length_or_warn(&txt, &mut warnings)), + thumbnail: video.thumbnail.into(), + channel: ChannelTag { + id: channel.id, + name: channel.name, + avatar: video + .channel_thumbnail_supported_renderers + .channel_thumbnail_with_link_renderer + .thumbnail + .into(), + verification: video.owner_badges.into(), + subscriber_count: None, + }, + publish_date: video.published_time_text.as_ref().and_then(|txt| { + timeago::parse_timeago_or_warn(lang, txt, &mut warnings) + }), + publish_date_txt: video.published_time_text, + view_count: video + .view_count_text + .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), + is_live: video.thumbnail_overlays.is_live(), + is_short: video.thumbnail_overlays.is_short(), + short_description: video + .detailed_metadata_snippets + .try_swap_remove(0) + .map(|s| s.snippet_text) + .unwrap_or_default(), + })), + Err(e) => { + warnings.push(e.to_string()); + None + } + } + } + response::search::SearchItem::PlaylistRenderer(mut playlist) => { + Some(SearchItem::Playlist(SearchPlaylist { + id: playlist.playlist_id, + name: playlist.title, + thumbnail: playlist + .thumbnails + .try_swap_remove(0) + .unwrap_or_default() + .into(), + video_count: playlist.video_count, + first_videos: playlist + .videos + .into_iter() + .map(|v| SearchPlaylistVideo { + id: v.child_video_renderer.video_id, + title: v.child_video_renderer.title, + length: v.child_video_renderer.length_text.and_then(|txt| { + util::parse_video_length_or_warn(&txt, &mut warnings) + }), + }) + .collect(), + })) + } + response::search::SearchItem::ChannelRenderer(channel) => { + Some(SearchItem::Channel(SearchChannel { + id: channel.channel_id, + name: channel.title, + avatar: channel.thumbnail.into(), + verification: channel.owner_badges.into(), + subscriber_count: channel + .subscriber_count_text + .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), + short_description: channel.description_snippet, + })) + } + response::search::SearchItem::ShowingResultsForRenderer { corrected_query } => { + c_query = Some(corrected_query); + None + } + response::search::SearchItem::None => None, + }) + .collect(); + ( + MapResult { + c: mapped_items, + warnings, + }, + c_query, + ) +} + +#[cfg(test)] +mod tests { + use crate::client::RustyPipe; + + #[tokio::test] + async fn t1() { + let rp = RustyPipe::builder().strict().build(); + let result = rp.query().search("doobydoobap").await.unwrap(); + dbg!(&result); + } +} diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 808f759..64a4e55 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -19,10 +19,10 @@ use super::{ }; #[derive(Debug, Serialize)] -struct QVideo { +struct QVideo<'a> { context: YTContext, /// YouTube video ID - video_id: String, + video_id: &'a str, /// Set to true to allow extraction of streams with sensitive content content_check_ok: bool, /// Probably refers to allowing sensitive content, too @@ -34,7 +34,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QVideo { context, - video_id: video_id.to_owned(), + video_id, content_check_ok: true, racy_check_ok: true, }; @@ -56,7 +56,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, - continuation: ctoken.to_owned(), + continuation: ctoken, }; self.execute_request::( @@ -73,7 +73,7 @@ impl RustyPipeQuery { let context = self.get_context(ClientType::Desktop, true).await; let request_body = QContinuation { context, - continuation: ctoken.to_owned(), + continuation: ctoken, }; self.execute_request::( @@ -181,7 +181,7 @@ impl MapResponse for response::VideoDetails { }; let comment_count = comment_count_section.and_then(|s| { - util::parse_large_numstr::( + util::parse_large_numstr::( &s.comments_entry_point_header_renderer.comment_count, lang, ) @@ -411,7 +411,7 @@ impl MapResponse> for response::VideoComments { count_text, } => { comment_count = count_text.and_then(|txt| { - util::parse_numeric_or_warn::(&txt, &mut warnings) + util::parse_numeric_or_warn::(&txt, &mut warnings) }); } }); diff --git a/src/download.rs b/src/download.rs index a9ce92f..7ba3fbf 100644 --- a/src/download.rs +++ b/src/download.rs @@ -56,22 +56,12 @@ fn parse_cr_header(cr_header: &str) -> Result<(u64, u64)> { ); Ok(( - captures - .get(2) - .unwrap() - .as_str() - .parse() - .or(Err(DownloadError::Progressive( - "could not parse range header number".into(), - )))?, - captures - .get(3) - .unwrap() - .as_str() - .parse() - .or(Err(DownloadError::Progressive( - "could not parse range header number".into(), - )))?, + captures.get(2).unwrap().as_str().parse().map_err(|_| { + DownloadError::Progressive("could not parse range header number".into()) + })?, + captures.get(3).unwrap().as_str().parse().map_err(|_| { + DownloadError::Progressive("could not parse range header number".into()) + })?, )) } @@ -96,7 +86,7 @@ async fn download_single_file>( // If the url is from googlevideo, extract file size from clen parameter let (url_base, url_params) = - util::url_to_params(url).or_else(|e| Err(DownloadError::Other(e.to_string().into())))?; + util::url_to_params(url).map_err(|e| DownloadError::Other(e.to_string().into()))?; let is_gvideo = url_base.ends_with(".googlevideo.com/videoplayback"); if is_gvideo { size = url_params.get("clen").and_then(|s| s.parse::().ok()); @@ -120,9 +110,9 @@ async fn download_single_file>( )) ) .to_str() - .or(Err(DownloadError::Progressive( - "could not convert Content-Range header to string".into(), - )))?; + .map_err(|_| { + DownloadError::Progressive("could not convert Content-Range header to string".into()) + })?; let (_, original_size) = parse_cr_header(cr_header)?; @@ -207,9 +197,9 @@ async fn download_chunks_by_header( )) ) .to_str() - .or(Err(DownloadError::Progressive( - "could not convert Content-Range header to string".into(), - )))?; + .map_err(|_| { + DownloadError::Progressive("could not convert Content-Range header to string".into()) + })?; let (parsed_offset, parsed_size) = parse_cr_header(cr_header)?; diff --git a/src/error.rs b/src/error.rs index 572fd9e..35875a5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -81,7 +81,7 @@ pub enum ExtractionError { InvalidData(Cow<'static, str>), #[error("got wrong result from YT: {0}")] WrongResult(String), - #[error("Warnings during deserialization/mapping")] + #[error("Warnings during deserialization/mapping in strict mode")] DeserializationWarnings, } diff --git a/src/model/mod.rs b/src/model/mod.rs index 3d0e9c3..5cce258 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -393,7 +393,7 @@ pub struct Playlist { /// Playlist videos pub videos: Paginator, /// Number of videos in the playlist - pub video_count: u32, + pub video_count: u64, /// Playlist thumbnail pub thumbnail: Vec, /// Playlist description in plaintext format @@ -607,7 +607,7 @@ pub struct Comment { /// Number of comment likes pub like_count: Option, /// Number of replies - pub reply_count: u32, + pub reply_count: u64, /// Paginator to fetch comment replies pub replies: Paginator, /// Is the comment from the channel owner? @@ -703,7 +703,7 @@ pub struct ChannelPlaylist { /// Playlist thumbnail pub thumbnail: Vec, /// Number of playlist videos - pub video_count: Option, + pub video_count: Option, } /// Additional channel metadata fetched from the "About" tab. @@ -753,5 +753,98 @@ pub struct ChannelRssVideo { /// Number of likes /// /// Zero if the like count was hidden by the creator. - pub like_count: u32, + pub like_count: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SearchResult { + pub items: Paginator, + pub corrected_query: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum SearchItem { + Video(SearchVideo), + Playlist(SearchPlaylist), + Channel(SearchChannel), +} + +/// YouTube video from the search results +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub struct SearchVideo { + /// Unique YouTube video ID + pub id: String, + /// Video title + pub title: String, + /// Video length in seconds. + /// + /// Is [`None`] for livestreams. + pub length: Option, + /// Video thumbnail + pub thumbnail: Vec, + /// Channel of the video + pub channel: ChannelTag, + /// Video publishing date. + /// + /// [`None`] if the date could not be parsed. + pub publish_date: Option>, + /// Textual video publish date (e.g. `11 months ago`, depends on language) + /// + /// Is [`None`] for livestreams. + pub publish_date_txt: Option, + /// View count + /// + /// [`None`] if it could not be extracted. + pub view_count: Option, + /// Is the video an active livestream? + pub is_live: bool, + /// Is the video a YouTube Short video (vertical and <60s)? + pub is_short: bool, + /// Abbreviated video description + pub short_description: String, +} + +/// Playlist from the search results +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub struct SearchPlaylist { + /// Unique YouTube Playlist-ID (e.g. `PL5dDx681T4bR7ZF1IuWzOv1omlRbE7PiJ`) + pub id: String, + /// Playlist name + pub name: String, + /// Playlist thumbnail + pub thumbnail: Vec, + /// Number of playlist videos + pub video_count: u64, + /// First 2 videos + pub first_videos: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub struct SearchPlaylistVideo { + pub id: String, + pub title: String, + pub length: Option, +} + +/// Channel from the search results +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[non_exhaustive] +pub struct SearchChannel { + /// Unique YouTube channel ID + pub id: String, + /// Channel name + pub name: String, + /// Channel avatar/profile picture + pub avatar: Vec, + /// Channel verification mark + pub verification: Verification, + /// Approximate number of subscribers + /// + /// [`None`] if hidden by the owner or not present. + pub subscriber_count: Option, + /// Abbreviated channel description + pub short_description: String, } diff --git a/src/model/paginator.rs b/src/model/paginator.rs index cc7766d..090e799 100644 --- a/src/model/paginator.rs +++ b/src/model/paginator.rs @@ -20,7 +20,7 @@ pub struct Paginator { /// /// Don't use this number to check if all items were fetched or for /// iterating over the items. - pub count: Option, + pub count: Option, /// Content of the paginator pub items: Vec, /// The continuation token is passed to the YouTube API to fetch @@ -41,7 +41,7 @@ impl Default for Paginator { } impl Paginator { - pub(crate) fn new(count: Option, items: Vec, ctoken: Option) -> Self { + pub(crate) fn new(count: Option, items: Vec, ctoken: Option) -> Self { Self { count: match ctoken { Some(_) => count, diff --git a/src/report.rs b/src/report.rs index 87e9d13..cf29592 100644 --- a/src/report.rs +++ b/src/report.rs @@ -98,11 +98,8 @@ impl FileReporter { fn _report(&self, report: &Report) -> Result<()> { let report_path = get_report_path(&self.path, report, "json")?; - serde_json::to_writer_pretty(&File::create(report_path)?, &report).or_else(|e| { - Err(Error::Other( - format!("could not serialize report. err: {}", e).into(), - )) - })?; + serde_json::to_writer_pretty(&File::create(report_path)?, &report) + .map_err(|e| Error::Other(format!("could not serialize report. err: {}", e).into()))?; Ok(()) } } diff --git a/src/serializer/text.rs b/src/serializer/text.rs index 3406836..dfa5321 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -305,23 +305,18 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText { } let mut buf = String::with_capacity(until - i_utf16); - loop { - match chars.next() { - Some(c) => { - buf.push(c); + for c in chars.by_ref() { + buf.push(c); - // is character on Basic Multilingual Plane -> 16bit in UTF-16, - // counts as 1 JS character, otherwise 32bit, counts as 2 JS characters - if (c as u32) > 0xffff { - i_utf16 += 1; - }; - i_utf16 += 1; + // is character on Basic Multilingual Plane -> 16bit in UTF-16, + // counts as 1 JS character, otherwise 32bit, counts as 2 JS characters + if (c as u32) > 0xffff { + i_utf16 += 1; + }; + i_utf16 += 1; - if i_utf16 >= until { - break; - } - } - None => break, + if i_utf16 >= until { + break; } } buf @@ -339,7 +334,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText { // Replace no-break spaces, trim off whitespace and prefix character let txt_link = txt_link.trim(); - let txt_link = txt_link.replace("\u{a0}", " "); + let txt_link = txt_link.replace('\u{a0}', " "); static LINK_PREFIX: Lazy = Lazy::new(|| Regex::new("^[/•] *").unwrap()); let txt_link = LINK_PREFIX.replace(&txt_link, ""); diff --git a/src/util.rs b/src/util/mod.rs similarity index 98% rename from src/util.rs rename to src/util/mod.rs index 67980a2..2928f63 100644 --- a/src/util.rs +++ b/src/util/mod.rs @@ -43,11 +43,8 @@ pub fn generate_content_playback_nonce() -> String { /// /// `example.com/api?k1=v1&k2=v2 => example.com/api; {k1: v1, k2: v2}` pub fn url_to_params(url: &str) -> Result<(String, BTreeMap)> { - let mut parsed_url = Url::parse(url).or_else(|e| { - Err(Error::Other( - format!("could not parse url `{}` err: {}", url, e).into(), - )) - })?; + let mut parsed_url = Url::parse(url) + .map_err(|e| Error::Other(format!("could not parse url `{}` err: {}", url, e).into()))?; let url_params: BTreeMap = parsed_url .query_pairs() .map(|(k, v)| (k.to_string(), v.to_string()))