Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add channel search

Browse source
This commit is contained in:
ThetaDev 2022-12-08 21:53:44 +01:00
parent 402f5834d2
commit 4c1876cb55
8 changed files with 151 additions and 171 deletions
Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -10,7 +10,7 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
- [X] **Player** (video/audio streams, subtitles)
- [X] **Playlist**
- [X] **VideoDetails** (metadata, comments, recommended videos)
- [X] **Channel** (videos, playlists, info)
- [X] **Channel** (videos, shorts, livestreams, playlists, info, search)
- [X] **ChannelRSS**
- [X] **Search** (with filters)
- [X] **Search suggestions**

167
src/client/channel.rs
View file

@ -1,22 +1,17 @@
use std::borrow::Cow;
use serde::Serialize;
use time::OffsetDateTime;
use url::Url;
use crate::{
error::{Error, ExtractionError},
model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem},
model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem, YouTubeItem},
param::Language,
serializer::MapResult,
timeago,
util::{self, TryRemove},
util,
};
use super::{
response::{self, channel::ChannelContent},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
@ -24,6 +19,8 @@ struct QChannel<'a> {
context: YTContext<'a>,
browse_id: &'a str,
params: Params,
#[serde(skip_serializing_if = "Option::is_none")]
query: Option<&'a str>,
}
#[derive(Debug, Serialize)]
@ -38,6 +35,8 @@ enum Params {
Playlists,
#[serde(rename = "EgVhYm91dPIGBAoCEgA%3D")]
Info,
#[serde(rename = "EgZzZWFyY2jyBgQKAloA")]
Search,
}
impl RustyPipeQuery {
@ -45,6 +44,7 @@ impl RustyPipeQuery {
&self,
channel_id: S,
params: Params,
query: Option<&str>,
operation: &str,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
let channel_id = channel_id.as_ref();
@ -53,6 +53,7 @@ impl RustyPipeQuery {
context,
browse_id: channel_id,
params,
query,
};
self.execute_request::<response::Channel, _, _>(
@ -69,7 +70,7 @@ impl RustyPipeQuery {
&self,
channel_id: S,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Videos, "channel_videos")
self._channel_videos(channel_id, Params::Videos, None, "channel_videos")
.await
}
@ -77,7 +78,7 @@ impl RustyPipeQuery {
&self,
channel_id: S,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Shorts, "channel_shorts")
self._channel_videos(channel_id, Params::Shorts, None, "channel_shorts")
.await
}
@ -85,10 +86,24 @@ impl RustyPipeQuery {
&self,
channel_id: S,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Live, "channel_livestreams")
self._channel_videos(channel_id, Params::Live, None, "channel_livestreams")
.await
}
pub async fn channel_search<S: AsRef<str>, S2: AsRef<str>>(
&self,
channel_id: S,
query: S2,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(
channel_id,
Params::Search,
Some(query.as_ref()),
"channel_search",
)
.await
}
pub async fn channel_playlists<S: AsRef<str>>(
&self,
channel_id: S,
@ -99,6 +114,7 @@ impl RustyPipeQuery {
context,
browse_id: channel_id,
params: Params::Playlists,
query: None,
};
self.execute_request::<response::Channel, _, _>(
@ -121,6 +137,7 @@ impl RustyPipeQuery {
context,
browse_id: channel_id,
params: Params::Info,
query: None,
};
self.execute_request::<response::Channel, _, _>(
@ -156,29 +173,20 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
lang,
)?;
let v_res = match content.content {
ChannelContent::GridRenderer { items } => {
let mut mapper =
response::YouTubeListMapper::<VideoItem>::with_channel(lang, &channel_data);
mapper.map_response(items);
MapResult {
c: Paginator::new_ext(
None,
mapper.items,
mapper.ctoken,
self.response_context.visitor_data,
crate::param::ContinuationEndpoint::Browse,
),
warnings: mapper.warnings,
}
}
_ => MapResult::default(),
};
let mut mapper =
response::YouTubeListMapper::<VideoItem>::with_channel(lang, &channel_data);
mapper.map_response(content.content);
let p = Paginator::new_ext(
None,
mapper.items,
mapper.ctoken,
self.response_context.visitor_data,
crate::param::ContinuationEndpoint::Browse,
);
Ok(MapResult {
c: combine_channel_data(channel_data, v_res.c),
warnings: v_res.warnings,
c: combine_channel_data(channel_data, p),
warnings: mapper.warnings,
})
}
}
@ -205,23 +213,14 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
lang,
)?;
let p_res = match content.content {
ChannelContent::GridRenderer { items } => {
let mut mapper =
response::YouTubeListMapper::<PlaylistItem>::with_channel(lang, &channel_data);
mapper.map_response(items);
MapResult {
c: Paginator::new(None, mapper.items, mapper.ctoken),
warnings: mapper.warnings,
}
}
_ => MapResult::default(),
};
let mut mapper =
response::YouTubeListMapper::<PlaylistItem>::with_channel(lang, &channel_data);
mapper.map_response(content.content);
let p = Paginator::new(None, mapper.items, mapper.ctoken);
Ok(MapResult {
c: combine_channel_data(channel_data, p_res.c),
warnings: p_res.warnings,
c: combine_channel_data(channel_data, p),
warnings: mapper.warnings,
})
}
}
@ -234,8 +233,6 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
let content = map_channel_content(self.contents, self.alerts)?;
let mut warnings = Vec::new();
let channel_data = map_channel(
MapChannelData {
header: self.header,
@ -249,38 +246,18 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
lang,
)?;
let cinfo = match content.content {
response::channel::ChannelContent::ChannelAboutFullMetadataRenderer(meta) => {
ChannelInfo {
create_date: timeago::parse_textual_date_or_warn(
lang,
&meta.joined_date_text,
&mut warnings,
)
.map(OffsetDateTime::date),
view_count: meta
.view_count_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut warnings)),
links: meta
.primary_links
.into_iter()
.filter_map(|l| {
l.navigation_endpoint
.url_endpoint
.map(|url| (l.title, util::sanitize_yt_url(&url.url)))
})
.collect(),
}
let mut mapper = response::YouTubeListMapper::<YouTubeItem>::new(lang);
mapper.map_response(content.content);
let mut warnings = mapper.warnings;
let cinfo = mapper.channel_info.unwrap_or_else(|| {
warnings.push("no aboutFullMetadata".to_owned());
ChannelInfo {
create_date: None,
view_count: None,
links: Vec::new(),
}
_ => {
warnings.push("no aboutFullMetadata".to_owned());
ChannelInfo {
create_date: None,
view_count: None,
links: Vec::new(),
}
}
};
});
Ok(MapResult {
c: combine_channel_data(channel_data, cinfo),
@ -406,7 +383,7 @@ fn map_channel(
}
struct MappedChannelContent {
content: response::channel::ChannelContent,
content: MapResult<Vec<response::YouTubeListItem>>,
has_shorts: bool,
has_live: bool,
}
@ -450,31 +427,19 @@ fn map_channel_content(
}
}
let channel_content = tabs
.into_iter()
.filter_map(|tab| {
let content = tab.tab_renderer.content;
match (content.rich_grid_renderer, content.section_list_renderer) {
(Some(rich_grid), _) => Some(ChannelContent::GridRenderer {
items: rich_grid.contents,
}),
(None, Some(section_list)) => {
let mut contents = section_list.contents;
contents.try_swap_remove(0).and_then(|mut i| {
i.item_section_renderer.contents.try_swap_remove(0)
})
}
(None, None) => None,
}
})
.next();
let channel_content = tabs.into_iter().find_map(|tab| {
tab.tab_renderer
.content
.rich_grid_renderer
.or(tab.tab_renderer.content.section_list_renderer)
});
let content = match channel_content {
Some(content) => content,
Some(list) => list.contents,
None => {
// YouTube may show the "Featured" tab if the requested tab is empty/does not exist
if featured_tab {
response::channel::ChannelContent::None
MapResult::default()
} else {
return Err(ExtractionError::InvalidData(Cow::Borrowed(
"could not extract content",

65
src/client/response/channel.rs
View file

@ -1,9 +1,11 @@
use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
use super::url_endpoint::NavigationEndpoint;
use super::{Alert, ChannelBadge, ContentsRenderer, ResponseContext, Thumbnails, YouTubeListItem};
use crate::serializer::{text::Text, MapResult, VecLogError};
use super::{
video_item::YouTubeListRenderer, Alert, ChannelBadge, ContentsRenderer, ResponseContext,
Thumbnails,
};
use crate::serializer::text::Text;
#[serde_as]
#[derive(Debug, Deserialize)]
@ -39,6 +41,7 @@ pub(crate) struct TabsRenderer {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TabRendererWrap {
#[serde(alias = "expandableTabRenderer")]
pub tab_renderer: TabRenderer,
}
@ -56,11 +59,10 @@ pub(crate) struct TabRenderer {
pub(crate) struct TabContent {
#[serde(default)]
#[serde_as(as = "DefaultOnError")]
pub section_list_renderer: Option<ContentsRenderer<ItemSectionRendererWrap>>,
/// Seems to be currently A/B tested, as of 11.10.2022
pub section_list_renderer: Option<YouTubeListRenderer>,
#[serde(default)]
#[serde_as(as = "DefaultOnError")]
pub rich_grid_renderer: Option<RichGridRenderer>,
pub rich_grid_renderer: Option<YouTubeListRenderer>,
}
#[derive(Debug, Deserialize)]
@ -81,35 +83,6 @@ pub(crate) struct ChannelTabWebCommandMetadata {
pub url: String,
}
/// Seems to be currently A/B tested, as of 11.10.2022
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RichGridRenderer {
#[serde_as(as = "VecLogError<_>")]
pub contents: MapResult<Vec<YouTubeListItem>>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ItemSectionRendererWrap {
pub item_section_renderer: ContentsRenderer<ChannelContent>,
}
#[serde_as]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) enum ChannelContent {
GridRenderer {
#[serde_as(as = "VecLogError<_>")]
items: MapResult<Vec<YouTubeListItem>>,
},
ChannelAboutFullMetadataRenderer(ChannelFullMetadata),
#[default]
#[serde(other, deserialize_with = "deserialize_ignore_any")]
None,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) enum Header {
@ -184,25 +157,3 @@ pub(crate) struct MicroformatDataRenderer {
#[serde(default)]
pub tags: Vec<String>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelFullMetadata {
#[serde_as(as = "Text")]
pub joined_date_text: String,
#[serde_as(as = "Option<Text>")]
pub view_count_text: Option<String>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub primary_links: Vec<PrimaryLink>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PrimaryLink {
#[serde_as(as = "Text")]
pub title: String,
pub navigation_endpoint: NavigationEndpoint,
}

12
src/client/response/mod.rs
View file

@ -53,6 +53,8 @@ use crate::error::ExtractionError;
use crate::serializer::MapResult;
use crate::serializer::{text::Text, VecLogError};
use self::video_item::YouTubeListRenderer;
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ContentRenderer<T> {
@ -215,15 +217,7 @@ pub(crate) struct ContinuationAction {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RichGridContinuationContents {
pub rich_grid_continuation: RichGridContinuation,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct RichGridContinuation {
#[serde_as(as = "VecLogError<_>")]
pub contents: MapResult<Vec<YouTubeListItem>>,
pub rich_grid_continuation: YouTubeListRenderer,
}
#[derive(Debug, Deserialize)]

61
src/client/response/video_item.rs
View file

@ -6,9 +6,12 @@ use serde_with::{
};
use time::{Duration, OffsetDateTime};
use super::{ChannelBadge, ContinuationEndpoint, Thumbnails};
use super::{url_endpoint::NavigationEndpoint, ChannelBadge, ContinuationEndpoint, Thumbnails};
use crate::{
model::{Channel, ChannelId, ChannelItem, ChannelTag, PlaylistItem, VideoItem, YouTubeItem},
model::{
Channel, ChannelId, ChannelInfo, ChannelItem, ChannelTag, PlaylistItem, VideoItem,
YouTubeItem,
},
param::Language,
serializer::{
text::{AccessibilityText, Text, TextComponent},
@ -45,6 +48,9 @@ pub(crate) enum YouTubeListItem {
corrected_query: String,
},
/// Channel metadata (about tab)
ChannelAboutFullMetadataRenderer(ChannelFullMetadata),
/// Contains video on startpage
///
/// Seems to be currently A/B tested on the channel page,
@ -58,7 +64,9 @@ pub(crate) enum YouTubeListItem {
///
/// Seems to be currently A/B tested on the video details page,
/// as of 11.10.2022
#[serde(alias = "expandedShelfContentsRenderer")]
///
/// GridRenderer: contains videos on channel page
#[serde(alias = "expandedShelfContentsRenderer", alias = "gridRenderer")]
ItemSectionRenderer {
#[serde(alias = "items")]
#[serde_as(as = "VecLogError<_>")]
@ -326,6 +334,28 @@ pub(crate) struct ReelPlayerHeaderRenderer {
pub timestamp_text: String,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelFullMetadata {
#[serde_as(as = "Text")]
pub joined_date_text: String,
#[serde_as(as = "Option<Text>")]
pub view_count_text: Option<String>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub primary_links: Vec<PrimaryLink>,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct PrimaryLink {
#[serde_as(as = "Text")]
pub title: String,
pub navigation_endpoint: NavigationEndpoint,
}
trait IsLive {
fn is_live(&self) -> bool;
}
@ -369,6 +399,7 @@ pub(crate) struct YouTubeListMapper<T> {
pub warnings: Vec<String>,
pub ctoken: Option<String>,
pub corrected_query: Option<String>,
pub channel_info: Option<ChannelInfo>,
}
impl<T> YouTubeListMapper<T> {
@ -380,6 +411,7 @@ impl<T> YouTubeListMapper<T> {
warnings: Vec::new(),
ctoken: None,
corrected_query: None,
channel_info: None,
}
}
@ -397,6 +429,7 @@ impl<T> YouTubeListMapper<T> {
warnings: Vec::new(),
ctoken: None,
corrected_query: None,
channel_info: None,
}
}
@ -581,6 +614,28 @@ impl YouTubeListMapper<YouTubeItem> {
YouTubeListItem::ShowingResultsForRenderer { corrected_query } => {
self.corrected_query = Some(corrected_query);
}
YouTubeListItem::ChannelAboutFullMetadataRenderer(meta) => {
self.channel_info = Some(ChannelInfo {
create_date: timeago::parse_textual_date_or_warn(
self.lang,
&meta.joined_date_text,
&mut self.warnings,
)
.map(OffsetDateTime::date),
view_count: meta
.view_count_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
links: meta
.primary_links
.into_iter()
.filter_map(|l| {
l.navigation_endpoint
.url_endpoint
.map(|url| (l.title, util::sanitize_yt_url(&url.url)))
})
.collect(),
})
}
YouTubeListItem::RichItemRenderer { content } => {
self.map_item(*content);
}

1
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap
View file

@ -37,6 +37,7 @@ Channel(
count: Some(0),
items: [],
ctoken: None,
visitor_data: Some("Cgtvc2s4UllvTGl6byigxseZBg%3D%3D"),
endpoint: browse,
),
)

1
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap
View file

@ -120,6 +120,7 @@ Channel(
count: Some(0),
items: [],
ctoken: None,
visitor_data: Some("CgtCV1l2R2Rzb2ZSZyiu4a2ZBg%3D%3D"),
endpoint: browse,
),
)

13
tests/youtube.rs
View file

@ -948,6 +948,19 @@ async fn channel_info() {
"###);
}
#[tokio::test]
async fn channel_search() {
let rp = RustyPipe::builder().strict().build();
let channel = rp
.query()
.channel_search("UC2DjFE7Xf11URZqWBigcVOQ", "test")
.await
.unwrap();
assert_channel_eevblog(&channel);
assert_next(channel.content, rp.query(), 20, 2).await;
}
fn assert_channel_eevblog<T>(channel: &Channel<T>) {
assert_eq!(channel.id, "UC2DjFE7Xf11URZqWBigcVOQ");
assert_eq!(channel.name, "EEVblog");