From 4c1876cb554c527e981c4220d77c9e72d83fdedd Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Thu, 8 Dec 2022 21:53:44 +0100 Subject: [PATCH] feat: add channel search --- README.md | 2 +- src/client/channel.rs | 167 +++++++----------- src/client/response/channel.rs | 65 +------ src/client/response/mod.rs | 12 +- src/client/response/video_item.rs | 61 ++++++- ...nnel__tests__map_channel_videos_empty.snap | 1 + ...nnel__tests__map_channel_videos_music.snap | 1 + tests/youtube.rs | 13 ++ 8 files changed, 151 insertions(+), 171 deletions(-) diff --git a/README.md b/README.md index dccea84..62df601 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). - [X] **Player** (video/audio streams, subtitles) - [X] **Playlist** - [X] **VideoDetails** (metadata, comments, recommended videos) -- [X] **Channel** (videos, playlists, info) +- [X] **Channel** (videos, shorts, livestreams, playlists, info, search) - [X] **ChannelRSS** - [X] **Search** (with filters) - [X] **Search suggestions** diff --git a/src/client/channel.rs b/src/client/channel.rs index 6094270..4bbac26 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -1,22 +1,17 @@ use std::borrow::Cow; use serde::Serialize; -use time::OffsetDateTime; use url::Url; use crate::{ error::{Error, ExtractionError}, - model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem}, + model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem, YouTubeItem}, param::Language, serializer::MapResult, - timeago, - util::{self, TryRemove}, + util, }; -use super::{ - response::{self, channel::ChannelContent}, - ClientType, MapResponse, RustyPipeQuery, YTContext, -}; +use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -24,6 +19,8 @@ struct QChannel<'a> { context: YTContext<'a>, browse_id: &'a str, params: Params, + #[serde(skip_serializing_if = "Option::is_none")] + query: Option<&'a str>, } #[derive(Debug, Serialize)] @@ -38,6 +35,8 @@ enum Params { Playlists, #[serde(rename = "EgVhYm91dPIGBAoCEgA%3D")] Info, + #[serde(rename = "EgZzZWFyY2jyBgQKAloA")] + Search, } impl RustyPipeQuery { @@ -45,6 +44,7 @@ impl RustyPipeQuery { &self, channel_id: S, params: Params, + query: Option<&str>, operation: &str, ) -> Result>, Error> { let channel_id = channel_id.as_ref(); @@ -53,6 +53,7 @@ impl RustyPipeQuery { context, browse_id: channel_id, params, + query, }; self.execute_request::( @@ -69,7 +70,7 @@ impl RustyPipeQuery { &self, channel_id: S, ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Videos, "channel_videos") + self._channel_videos(channel_id, Params::Videos, None, "channel_videos") .await } @@ -77,7 +78,7 @@ impl RustyPipeQuery { &self, channel_id: S, ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Shorts, "channel_shorts") + self._channel_videos(channel_id, Params::Shorts, None, "channel_shorts") .await } @@ -85,10 +86,24 @@ impl RustyPipeQuery { &self, channel_id: S, ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Live, "channel_livestreams") + self._channel_videos(channel_id, Params::Live, None, "channel_livestreams") .await } + pub async fn channel_search, S2: AsRef>( + &self, + channel_id: S, + query: S2, + ) -> Result>, Error> { + self._channel_videos( + channel_id, + Params::Search, + Some(query.as_ref()), + "channel_search", + ) + .await + } + pub async fn channel_playlists>( &self, channel_id: S, @@ -99,6 +114,7 @@ impl RustyPipeQuery { context, browse_id: channel_id, params: Params::Playlists, + query: None, }; self.execute_request::( @@ -121,6 +137,7 @@ impl RustyPipeQuery { context, browse_id: channel_id, params: Params::Info, + query: None, }; self.execute_request::( @@ -156,29 +173,20 @@ impl MapResponse>> for response::Channel { lang, )?; - let v_res = match content.content { - ChannelContent::GridRenderer { items } => { - let mut mapper = - response::YouTubeListMapper::::with_channel(lang, &channel_data); - mapper.map_response(items); - - MapResult { - c: Paginator::new_ext( - None, - mapper.items, - mapper.ctoken, - self.response_context.visitor_data, - crate::param::ContinuationEndpoint::Browse, - ), - warnings: mapper.warnings, - } - } - _ => MapResult::default(), - }; + let mut mapper = + response::YouTubeListMapper::::with_channel(lang, &channel_data); + mapper.map_response(content.content); + let p = Paginator::new_ext( + None, + mapper.items, + mapper.ctoken, + self.response_context.visitor_data, + crate::param::ContinuationEndpoint::Browse, + ); Ok(MapResult { - c: combine_channel_data(channel_data, v_res.c), - warnings: v_res.warnings, + c: combine_channel_data(channel_data, p), + warnings: mapper.warnings, }) } } @@ -205,23 +213,14 @@ impl MapResponse>> for response::Channel { lang, )?; - let p_res = match content.content { - ChannelContent::GridRenderer { items } => { - let mut mapper = - response::YouTubeListMapper::::with_channel(lang, &channel_data); - mapper.map_response(items); - - MapResult { - c: Paginator::new(None, mapper.items, mapper.ctoken), - warnings: mapper.warnings, - } - } - _ => MapResult::default(), - }; + let mut mapper = + response::YouTubeListMapper::::with_channel(lang, &channel_data); + mapper.map_response(content.content); + let p = Paginator::new(None, mapper.items, mapper.ctoken); Ok(MapResult { - c: combine_channel_data(channel_data, p_res.c), - warnings: p_res.warnings, + c: combine_channel_data(channel_data, p), + warnings: mapper.warnings, }) } } @@ -234,8 +233,6 @@ impl MapResponse> for response::Channel { _deobf: Option<&crate::deobfuscate::Deobfuscator>, ) -> Result>, ExtractionError> { let content = map_channel_content(self.contents, self.alerts)?; - let mut warnings = Vec::new(); - let channel_data = map_channel( MapChannelData { header: self.header, @@ -249,38 +246,18 @@ impl MapResponse> for response::Channel { lang, )?; - let cinfo = match content.content { - response::channel::ChannelContent::ChannelAboutFullMetadataRenderer(meta) => { - ChannelInfo { - create_date: timeago::parse_textual_date_or_warn( - lang, - &meta.joined_date_text, - &mut warnings, - ) - .map(OffsetDateTime::date), - view_count: meta - .view_count_text - .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)), - links: meta - .primary_links - .into_iter() - .filter_map(|l| { - l.navigation_endpoint - .url_endpoint - .map(|url| (l.title, util::sanitize_yt_url(&url.url))) - }) - .collect(), - } + let mut mapper = response::YouTubeListMapper::::new(lang); + mapper.map_response(content.content); + let mut warnings = mapper.warnings; + + let cinfo = mapper.channel_info.unwrap_or_else(|| { + warnings.push("no aboutFullMetadata".to_owned()); + ChannelInfo { + create_date: None, + view_count: None, + links: Vec::new(), } - _ => { - warnings.push("no aboutFullMetadata".to_owned()); - ChannelInfo { - create_date: None, - view_count: None, - links: Vec::new(), - } - } - }; + }); Ok(MapResult { c: combine_channel_data(channel_data, cinfo), @@ -406,7 +383,7 @@ fn map_channel( } struct MappedChannelContent { - content: response::channel::ChannelContent, + content: MapResult>, has_shorts: bool, has_live: bool, } @@ -450,31 +427,19 @@ fn map_channel_content( } } - let channel_content = tabs - .into_iter() - .filter_map(|tab| { - let content = tab.tab_renderer.content; - match (content.rich_grid_renderer, content.section_list_renderer) { - (Some(rich_grid), _) => Some(ChannelContent::GridRenderer { - items: rich_grid.contents, - }), - (None, Some(section_list)) => { - let mut contents = section_list.contents; - contents.try_swap_remove(0).and_then(|mut i| { - i.item_section_renderer.contents.try_swap_remove(0) - }) - } - (None, None) => None, - } - }) - .next(); + let channel_content = tabs.into_iter().find_map(|tab| { + tab.tab_renderer + .content + .rich_grid_renderer + .or(tab.tab_renderer.content.section_list_renderer) + }); let content = match channel_content { - Some(content) => content, + Some(list) => list.contents, None => { // YouTube may show the "Featured" tab if the requested tab is empty/does not exist if featured_tab { - response::channel::ChannelContent::None + MapResult::default() } else { return Err(ExtractionError::InvalidData(Cow::Borrowed( "could not extract content", diff --git a/src/client/response/channel.rs b/src/client/response/channel.rs index a8abe40..673c86f 100644 --- a/src/client/response/channel.rs +++ b/src/client/response/channel.rs @@ -1,9 +1,11 @@ use serde::Deserialize; use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError}; -use super::url_endpoint::NavigationEndpoint; -use super::{Alert, ChannelBadge, ContentsRenderer, ResponseContext, Thumbnails, YouTubeListItem}; -use crate::serializer::{text::Text, MapResult, VecLogError}; +use super::{ + video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ResponseContext, + Thumbnails, +}; +use crate::serializer::text::Text; #[serde_as] #[derive(Debug, Deserialize)] @@ -39,6 +41,7 @@ pub(crate) struct TabsRenderer { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct TabRendererWrap { + #[serde(alias = "expandableTabRenderer")] pub tab_renderer: TabRenderer, } @@ -56,11 +59,10 @@ pub(crate) struct TabRenderer { pub(crate) struct TabContent { #[serde(default)] #[serde_as(as = "DefaultOnError")] - pub section_list_renderer: Option>, - /// Seems to be currently A/B tested, as of 11.10.2022 + pub section_list_renderer: Option, #[serde(default)] #[serde_as(as = "DefaultOnError")] - pub rich_grid_renderer: Option, + pub rich_grid_renderer: Option, } #[derive(Debug, Deserialize)] @@ -81,35 +83,6 @@ pub(crate) struct ChannelTabWebCommandMetadata { pub url: String, } -/// Seems to be currently A/B tested, as of 11.10.2022 -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct RichGridRenderer { - #[serde_as(as = "VecLogError<_>")] - pub contents: MapResult>, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ItemSectionRendererWrap { - pub item_section_renderer: ContentsRenderer, -} - -#[serde_as] -#[derive(Default, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) enum ChannelContent { - GridRenderer { - #[serde_as(as = "VecLogError<_>")] - items: MapResult>, - }, - ChannelAboutFullMetadataRenderer(ChannelFullMetadata), - #[default] - #[serde(other, deserialize_with = "deserialize_ignore_any")] - None, -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) enum Header { @@ -184,25 +157,3 @@ pub(crate) struct MicroformatDataRenderer { #[serde(default)] pub tags: Vec, } - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ChannelFullMetadata { - #[serde_as(as = "Text")] - pub joined_date_text: String, - #[serde_as(as = "Option")] - pub view_count_text: Option, - #[serde(default)] - #[serde_as(as = "VecSkipError<_>")] - pub primary_links: Vec, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct PrimaryLink { - #[serde_as(as = "Text")] - pub title: String, - pub navigation_endpoint: NavigationEndpoint, -} diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index 1534240..82d7d6d 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -53,6 +53,8 @@ use crate::error::ExtractionError; use crate::serializer::MapResult; use crate::serializer::{text::Text, VecLogError}; +use self::video_item::YouTubeListRenderer; + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ContentRenderer { @@ -215,15 +217,7 @@ pub(crate) struct ContinuationAction { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct RichGridContinuationContents { - pub rich_grid_continuation: RichGridContinuation, -} - -#[serde_as] -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct RichGridContinuation { - #[serde_as(as = "VecLogError<_>")] - pub contents: MapResult>, + pub rich_grid_continuation: YouTubeListRenderer, } #[derive(Debug, Deserialize)] diff --git a/src/client/response/video_item.rs b/src/client/response/video_item.rs index 77184a0..a59346d 100644 --- a/src/client/response/video_item.rs +++ b/src/client/response/video_item.rs @@ -6,9 +6,12 @@ use serde_with::{ }; use time::{Duration, OffsetDateTime}; -use super::{ChannelBadge, ContinuationEndpoint, Thumbnails}; +use super::{url_endpoint::NavigationEndpoint, ChannelBadge, ContinuationEndpoint, Thumbnails}; use crate::{ - model::{Channel, ChannelId, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem}, + model::{ + Channel, ChannelId, ChannelInfo, ChannelItem, ChannelTag, PlaylistItem, VideoItem, + YouTubeItem, + }, param::Language, serializer::{ text::{AccessibilityText, Text, TextComponent}, @@ -45,6 +48,9 @@ pub(crate) enum YouTubeListItem { corrected_query: String, }, + /// Channel metadata (about tab) + ChannelAboutFullMetadataRenderer(ChannelFullMetadata), + /// Contains video on startpage /// /// Seems to be currently A/B tested on the channel page, @@ -58,7 +64,9 @@ pub(crate) enum YouTubeListItem { /// /// Seems to be currently A/B tested on the video details page, /// as of 11.10.2022 - #[serde(alias = "expandedShelfContentsRenderer")] + /// + /// GridRenderer: contains videos on channel page + #[serde(alias = "expandedShelfContentsRenderer", alias = "gridRenderer")] ItemSectionRenderer { #[serde(alias = "items")] #[serde_as(as = "VecLogError<_>")] @@ -326,6 +334,28 @@ pub(crate) struct ReelPlayerHeaderRenderer { pub timestamp_text: String, } +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ChannelFullMetadata { + #[serde_as(as = "Text")] + pub joined_date_text: String, + #[serde_as(as = "Option")] + pub view_count_text: Option, + #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] + pub primary_links: Vec, +} + +#[serde_as] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct PrimaryLink { + #[serde_as(as = "Text")] + pub title: String, + pub navigation_endpoint: NavigationEndpoint, +} + trait IsLive { fn is_live(&self) -> bool; } @@ -369,6 +399,7 @@ pub(crate) struct YouTubeListMapper { pub warnings: Vec, pub ctoken: Option, pub corrected_query: Option, + pub channel_info: Option, } impl YouTubeListMapper { @@ -380,6 +411,7 @@ impl YouTubeListMapper { warnings: Vec::new(), ctoken: None, corrected_query: None, + channel_info: None, } } @@ -397,6 +429,7 @@ impl YouTubeListMapper { warnings: Vec::new(), ctoken: None, corrected_query: None, + channel_info: None, } } @@ -581,6 +614,28 @@ impl YouTubeListMapper { YouTubeListItem::ShowingResultsForRenderer { corrected_query } => { self.corrected_query = Some(corrected_query); } + YouTubeListItem::ChannelAboutFullMetadataRenderer(meta) => { + self.channel_info = Some(ChannelInfo { + create_date: timeago::parse_textual_date_or_warn( + self.lang, + &meta.joined_date_text, + &mut self.warnings, + ) + .map(OffsetDateTime::date), + view_count: meta + .view_count_text + .and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)), + links: meta + .primary_links + .into_iter() + .filter_map(|l| { + l.navigation_endpoint + .url_endpoint + .map(|url| (l.title, util::sanitize_yt_url(&url.url))) + }) + .collect(), + }) + } YouTubeListItem::RichItemRenderer { content } => { self.map_item(*content); } diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap index e2c829a..780e6a6 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap @@ -37,6 +37,7 @@ Channel( count: Some(0), items: [], ctoken: None, + visitor_data: Some("Cgtvc2s4UllvTGl6byigxseZBg%3D%3D"), endpoint: browse, ), ) diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap index ea67ed9..4d7ed05 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap @@ -120,6 +120,7 @@ Channel( count: Some(0), items: [], ctoken: None, + visitor_data: Some("CgtCV1l2R2Rzb2ZSZyiu4a2ZBg%3D%3D"), endpoint: browse, ), ) diff --git a/tests/youtube.rs b/tests/youtube.rs index 8462b77..8c4ccb2 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -948,6 +948,19 @@ async fn channel_info() { "###); } +#[tokio::test] +async fn channel_search() { + let rp = RustyPipe::builder().strict().build(); + let channel = rp + .query() + .channel_search("UC2DjFE7Xf11URZqWBigcVOQ", "test") + .await + .unwrap(); + + assert_channel_eevblog(&channel); + assert_next(channel.content, rp.query(), 20, 2).await; +} + fn assert_channel_eevblog(channel: &Channel) { assert_eq!(channel.id, "UC2DjFE7Xf11URZqWBigcVOQ"); assert_eq!(channel.name, "EEVblog");