Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add has_shorts/has_live info to channels

Browse source
This commit is contained in:
ThetaDev 2022-10-28 21:52:16 +02:00
parent 1c0c64a8bf
commit 17f71dc9f5
13 changed files with 213 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

227
src/client/channel.rs
View file

@ -13,7 +13,10 @@ use crate::{
util::{self, TryRemove},
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
use super::{
response::{self, channel::ChannelContent},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
@ -33,6 +36,27 @@ enum Params {
Info,
}
#[derive(Debug, Clone, Copy)]
enum ChannelTab {
Videos,
Shorts,
Live,
Playlists,
Info,
}
impl ChannelTab {
const fn url_suffix(self) -> &'static str {
match self {
ChannelTab::Videos => "/videos",
ChannelTab::Shorts => "/shorts",
ChannelTab::Live => "/streams",
ChannelTab::Playlists => "/playlists",
ChannelTab::Info => "/about",
}
}
}
impl RustyPipeQuery {
pub async fn channel_videos(
&self,
@ -102,8 +126,8 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<Paginator<VideoItem>>>, ExtractionError> {
let content = map_channel_content(self.contents, id, self.alerts)?;
let grid = match content {
let content = map_channel_content(self.contents, ChannelTab::Videos, self.alerts)?;
let grid = match content.content {
response::channel::ChannelContent::GridRenderer { items } => Some(items),
_ => None,
};
@ -112,11 +136,15 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
Ok(MapResult {
c: map_channel(
self.header,
self.metadata,
self.microformat,
self.response_context.visitor_data,
v_res.c,
MapChannelData {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data,
has_shorts: content.has_shorts,
has_live: content.has_live,
content: v_res.c,
},
id,
lang,
)?,
@ -132,8 +160,8 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<Paginator<PlaylistItem>>>, ExtractionError> {
let content = map_channel_content(self.contents, id, self.alerts)?;
let grid = match content {
let content = map_channel_content(self.contents, ChannelTab::Playlists, self.alerts)?;
let grid = match content.content {
response::channel::ChannelContent::GridRenderer { items } => Some(items),
_ => None,
};
@ -144,11 +172,15 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
Ok(MapResult {
c: map_channel(
self.header,
self.metadata,
self.microformat,
self.response_context.visitor_data,
p_res.c,
MapChannelData {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data,
has_shorts: content.has_shorts,
has_live: content.has_live,
content: p_res.c,
},
id,
lang,
)?,
@ -164,9 +196,9 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
lang: Language,
_deobf: Option<&crate::deobfuscate::Deobfuscator>,
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
let content = map_channel_content(self.contents, id, self.alerts)?;
let content = map_channel_content(self.contents, ChannelTab::Info, self.alerts)?;
let mut warnings = Vec::new();
let meta = match content {
let meta = match content.content {
response::channel::ChannelContent::ChannelAboutFullMetadataRenderer(meta) => Some(meta),
_ => None,
};
@ -203,11 +235,15 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
Ok(MapResult {
c: map_channel(
self.header,
self.metadata,
self.microformat,
self.response_context.visitor_data,
cinfo,
MapChannelData {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data,
has_shorts: content.has_shorts,
has_live: content.has_live,
content: cinfo,
},
id,
lang,
)?,
@ -254,26 +290,37 @@ fn map_vanity_url(url: &str, id: &str) -> Option<String> {
})
}
fn map_channel<T>(
struct MapChannelData<T> {
header: Option<response::channel::Header>,
metadata: Option<response::channel::Metadata>,
microformat: Option<response::channel::Microformat>,
visitor_data: Option<String>,
has_shorts: bool,
has_live: bool,
content: T,
}
fn map_channel<T>(
d: MapChannelData<T>,
id: &str,
lang: Language,
) -> Result<Channel<T>, ExtractionError> {
let header = header.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
"channel not found",
)))?;
let metadata = metadata
let header = d
.header
.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
"channel not found",
)))?;
let metadata = d
.metadata
.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
"channel not found",
)))?
.channel_metadata_renderer;
let microformat = microformat.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
"channel not found",
)))?;
let microformat = d
.microformat
.ok_or(ExtractionError::ContentUnavailable(Cow::Borrowed(
"channel not found",
)))?;
if metadata.external_id != id {
return Err(ExtractionError::WrongResult(format!(
@ -302,8 +349,10 @@ fn map_channel<T>(
banner: header.banner.into(),
mobile_banner: header.mobile_banner.into(),
tv_banner: header.tv_banner.into(),
visitor_data,
content,
has_shorts: d.has_shorts,
has_live: d.has_live,
visitor_data: d.visitor_data,
content: d.content,
},
response::channel::Header::CarouselHeaderRenderer(carousel) => {
let hdata = carousel
@ -337,18 +386,26 @@ fn map_channel<T>(
banner: Vec::new(),
mobile_banner: Vec::new(),
tv_banner: Vec::new(),
visitor_data,
content,
has_shorts: d.has_shorts,
has_live: d.has_live,
visitor_data: d.visitor_data,
content: d.content,
}
}
})
}
struct MappedChannelContent {
content: response::channel::ChannelContent,
has_shorts: bool,
has_live: bool,
}
fn map_channel_content(
contents: Option<response::channel::Contents>,
id: &str,
channel_tab: ChannelTab,
alerts: Option<Vec<response::Alert>>,
) -> Result<response::channel::ChannelContent, ExtractionError> {
) -> Result<MappedChannelContent, ExtractionError> {
match contents {
Some(contents) => {
let tabs = contents.two_column_browse_results_renderer.tabs;
@ -358,42 +415,78 @@ fn map_channel_content(
));
}
let (channel_content, target_id) = tabs
.into_iter()
.filter_map(|tab| {
let content = tab.tab_renderer.content;
match (content.section_list_renderer, content.rich_grid_renderer) {
(Some(mut section_list_renderer), _) => {
let content =
section_list_renderer.contents.try_swap_remove(0).and_then(
|mut i| i.item_section_renderer.contents.try_swap_remove(0),
);
let cmp_url_suffix = |endpoint: &response::channel::ChannelTabEndpoint,
expect: &str| {
endpoint
.command_metadata
.web_command_metadata
.url
.ends_with(expect)
};
content.map(|c| (c, section_list_renderer.target_id))
}
(None, Some(rich_grid_renderer)) => Some((
response::channel::ChannelContent::GridRenderer {
items: rich_grid_renderer.contents,
},
rich_grid_renderer.target_id,
)),
(None, None) => None,
}
})
.next()
.ok_or(ExtractionError::InvalidData(Cow::Borrowed(
"could not extract content",
)))?;
let mut has_shorts = false;
let mut has_live = false;
let mut featured_tab = false;
if let Some(target_id) = target_id {
// YouTube falls back to the featured page if the channel does not have a "videos" tab.
// This is the case for YouTube Music channels.
if target_id.starts_with(&format!("browse-feed{}featured", id)) {
return Ok(response::channel::ChannelContent::None);
for tab in &tabs {
if cmp_url_suffix(&tab.tab_renderer.endpoint, "/featured")
&& (tab.tab_renderer.content.section_list_renderer.is_some()
|| tab.tab_renderer.content.rich_grid_renderer.is_some())
{
featured_tab = true;
} else if cmp_url_suffix(
&tab.tab_renderer.endpoint,
ChannelTab::Shorts.url_suffix(),
) {
has_shorts = true;
} else if cmp_url_suffix(&tab.tab_renderer.endpoint, ChannelTab::Live.url_suffix())
{
has_live = true;
}
}
Ok(channel_content)
let channel_content = tabs
.into_iter()
.filter_map(|tab| {
if cmp_url_suffix(&tab.tab_renderer.endpoint, channel_tab.url_suffix()) {
let content = tab.tab_renderer.content;
match (content.rich_grid_renderer, content.section_list_renderer) {
(Some(rich_grid), _) => Some(ChannelContent::GridRenderer {
items: rich_grid.contents,
}),
(None, Some(section_list)) => {
let mut contents = section_list.contents;
contents.try_swap_remove(0).and_then(|mut i| {
i.item_section_renderer.contents.try_swap_remove(0)
})
}
(None, None) => None,
}
} else {
None
}
})
.next();
let content = match channel_content {
Some(content) => content,
None => {
// YouTube may show the "Featured" tab if the requested tab is empty/does not exist
if featured_tab {
response::channel::ChannelContent::None
} else {
return Err(ExtractionError::InvalidData(Cow::Borrowed(
"could not extract content",
)));
}
}
};
Ok(MappedChannelContent {
content,
has_shorts,
has_live,
})
}
None => Err(response::alerts_to_err(alerts)),
}

42
src/client/response/channel.rs
View file

@ -3,9 +3,7 @@ use serde_with::serde_as;
use serde_with::{DefaultOnError, VecSkipError};
use super::url_endpoint::NavigationEndpoint;
use super::{Alert, ChannelBadge, ResponseContext};
use super::{ContentRenderer, ContentsRenderer};
use super::{Thumbnails, YouTubeListItem};
use super::{Alert, ChannelBadge, ContentsRenderer, ResponseContext, Thumbnails, YouTubeListItem};
use crate::serializer::ignore_any;
use crate::serializer::{text::Text, MapResult, VecLogError};
@ -43,11 +41,19 @@ pub(crate) struct TabsRenderer {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TabRendererWrap {
pub tab_renderer: ContentRenderer<TabContent>,
pub tab_renderer: TabRenderer,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TabRenderer {
#[serde(default)]
pub content: TabContent,
pub endpoint: ChannelTabEndpoint,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[derive(Default, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct TabContent {
#[serde(default)]
@ -59,14 +65,28 @@ pub(crate) struct TabContent {
pub rich_grid_renderer: Option<RichGridRenderer>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelTabEndpoint {
pub command_metadata: ChannelTabCommandMetadata,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelTabCommandMetadata {
pub web_command_metadata: ChannelTabWebCommandMetadata,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ChannelTabWebCommandMetadata {
pub url: String,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct SectionListRenderer {
pub contents: Vec<ItemSectionRendererWrap>,
/// - **Videos**: browse-feedUC2DjFE7Xf11URZqWBigcVOQvideos (...)
/// - **Playlists**: browse-feedUC2DjFE7Xf11URZqWBigcVOQplaylists104 (...)
/// - **Info**: None
pub target_id: Option<String>,
}
/// Seems to be currently A/B tested, as of 11.10.2022
@ -76,10 +96,6 @@ pub(crate) struct SectionListRenderer {
pub(crate) struct RichGridRenderer {
#[serde_as(as = "VecLogError<_>")]
pub contents: MapResult<Vec<YouTubeListItem>>,
/// - **Videos**: browse-feedUC2DjFE7Xf11URZqWBigcVOQvideos (...)
/// - **Playlists**: browse-feedUC2DjFE7Xf11URZqWBigcVOQplaylists104 (...)
/// - **Info**: None
pub target_id: Option<String>,
}
#[derive(Debug, Deserialize)]

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_info.snap
View file

@ -142,6 +142,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgszMUUzZDlGLWxiRSipqr2ZBg%3D%3D"),
content: ChannelInfo(
create_date: Some("2009-04-04"),

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_playlists.snap
View file

@ -142,6 +142,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgttaWpyTVpUN1AyZyioqr2ZBg%3D%3D"),
content: Paginator(
count: None,

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid.snap
View file

@ -113,6 +113,8 @@ Channel(
height: 1192,
),
],
has_shorts: true,
has_live: false,
visitor_data: Some("CgtQdE9zVVR3NVBDbyjz0ZKaBg%3D%3D"),
content: Paginator(
count: None,

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_20221011_richgrid2.snap
View file

@ -142,6 +142,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: true,
visitor_data: Some("Cgs4ZFVmMzVlU1dxbyiBqpeaBg%3D%3D"),
content: Paginator(
count: None,

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_base.snap
View file

@ -142,6 +142,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgszNU5rbDVZS2hMcyim4K2ZBg%3D%3D"),
content: Paginator(
count: None,

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_empty.snap
View file

@ -30,6 +30,8 @@ Channel(
banner: [],
mobile_banner: [],
tv_banner: [],
has_shorts: false,
has_live: false,
visitor_data: Some("Cgtvc2s4UllvTGl6byigxseZBg%3D%3D"),
content: Paginator(
count: Some(0),

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap
View file

@ -126,6 +126,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgtkYXJITElwYmd4OCj85a2ZBg%3D%3D"),
content: Paginator(
count: Some(21),

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_music.snap
View file

@ -113,6 +113,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgtCV1l2R2Rzb2ZSZyiu4a2ZBg%3D%3D"),
content: Paginator(
count: Some(0),

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap
View file

@ -113,6 +113,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("CgtneXVRbGtSMWtlYyj75a2ZBg%3D%3D"),
content: Paginator(
count: None,

2
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_upcoming.snap
View file

@ -130,6 +130,8 @@ Channel(
height: 1192,
),
],
has_shorts: false,
has_live: false,
visitor_data: Some("Cgs4Ri1tLW1KNWozNCjGk8yZBg%3D%3D"),
content: Paginator(
count: None,

4
src/model/mod.rs
View file

@ -683,6 +683,10 @@ pub struct Channel<T> {
pub mobile_banner: Vec<Thumbnail>,
/// Banner image shown above the channel (16:9 fullscreen format for TV)
pub tv_banner: Vec<Thumbnail>,
/// Does the channel have a *Shorts* tab?
pub has_shorts: bool,
/// Does the channel have a *Live* tab?
pub has_live: bool,
/// YouTube visitor data cookie
pub visitor_data: Option<String>,
/// Content fetched from the channel