Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: handling new podcast links

Browse source
This commit is contained in:
ThetaDev 2023-11-04 01:14:54 +01:00
parent ba06e2c8c8
commit 452f765ffd
11 changed files with 147 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/client/music_artist.rs
View file

@ -235,7 +235,6 @@ fn map_artist_page(
}
}
mapper.check_unknown()?;
let mut mapped = mapper.group_items();
static WIKIPEDIA_REGEX: Lazy<Regex> =
@ -332,7 +331,6 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
mapper.map_response(grid.grid_renderer.items);
}
mapper.check_unknown()?;
let mapped = mapper.group_items();
Ok(MapResult {

5
src/client/music_charts.rs
View file

@ -98,6 +98,7 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
h.music_carousel_shelf_basic_header_renderer
.more_content_button
.and_then(|btn| btn.button_renderer.navigation_endpoint.music_page())
.map(|mp| (mp.typ, mp.id))
}) {
Some((MusicPageType::Playlist, id)) => {
// Top music videos (first shelf with associated playlist)
@ -120,10 +121,6 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
response::music_charts::ItemSection::None => {}
});
mapper_top.check_unknown()?;
mapper_trending.check_unknown()?;
mapper_other.check_unknown()?;
let mapped_top = mapper_top.conv_items::<TrackItem>();
let mut mapped_trending = mapper_trending.conv_items::<TrackItem>();
let mut mapped_other = mapper_other.group_items();

3
src/client/music_details.rs
View file

@ -387,9 +387,6 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
_ => {}
});
mapper.check_unknown()?;
mapper_tracks.check_unknown()?;
let mapped_tracks = mapper_tracks.conv_items();
let mut mapped = mapper.group_items();

1
src/client/music_new.rs
View file

@ -75,7 +75,6 @@ impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
let mut mapper = MusicListMapper::new(lang);
mapper.map_response(items);
mapper.check_unknown()?;
Ok(mapper.conv_items())
}

1
src/client/music_playlist.rs
View file

@ -174,7 +174,6 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
let mut mapper = MusicListMapper::new(lang);
mapper.map_response(shelf.contents);
mapper.check_unknown()?;
let map_res = mapper.conv_items();
let ctoken = shelf

3
src/client/music_search.rs
View file

@ -266,7 +266,6 @@ impl MapResponse<MusicSearchResult> for response::MusicSearch {
response::music_search::ItemSection::None => {}
});
mapper.check_unknown()?;
let map_res = mapper.group_items();
Ok(MapResult {
@ -325,7 +324,6 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
response::music_search::ItemSection::None => {}
});
mapper.check_unknown()?;
let map_res = mapper.conv_items();
Ok(MapResult {
@ -371,7 +369,6 @@ impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
}
}
mapper.check_unknown()?;
let map_res = mapper.conv_items();
Ok(MapResult {

121
src/client/response/music_item.rs
View file

@ -2,14 +2,13 @@ use serde::Deserialize;
use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError};
use crate::{
error::ExtractionError,
model::{
self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId,
MusicItem, MusicItemType, MusicPlaylistItem, TrackItem,
},
param::Language,
serializer::{
text::{Text, TextComponents},
text::{Text, TextComponent, TextComponents},
MapResult,
},
util::{self, dictionary},
@ -17,7 +16,7 @@ use crate::{
use super::{
url_endpoint::{
BrowseEndpointWrap, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
BrowseEndpointWrap, MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
},
ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap,
};
@ -434,8 +433,6 @@ pub(crate) struct MusicListMapper {
search_suggestion: bool,
items: Vec<MusicItem>,
warnings: Vec<String>,
/// True if unknown items were mapped
has_unknown: bool,
}
#[derive(Debug)]
@ -456,7 +453,6 @@ impl MusicListMapper {
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
}
}
@ -469,7 +465,6 @@ impl MusicListMapper {
search_suggestion: true,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
}
}
@ -483,7 +478,6 @@ impl MusicListMapper {
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
}
}
@ -497,7 +491,6 @@ impl MusicListMapper {
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
}
}
@ -545,55 +538,44 @@ impl MusicListMapper {
.thumbnails
.first();
let pt_id = item
let music_page = item
.navigation_endpoint
.and_then(NavigationEndpoint::music_page)
.or_else(|| {
c1.and_then(|c1| {
c1.renderer.text.0.into_iter().next().and_then(|t| match t {
crate::serializer::text::TextComponent::Video {
video_id, vtype, ..
} => Some((MusicPageType::Track { vtype }, video_id)),
crate::serializer::text::TextComponent::Browse {
page_type,
browse_id,
..
} => Some((page_type.into(), browse_id)),
_ => None,
})
c1.renderer
.text
.0
.into_iter()
.next()
.and_then(TextComponent::music_page)
})
})
.or_else(|| {
item.playlist_item_data.map(|d| {
(
MusicPageType::Track {
vtype: MusicVideoType::from_is_video(
self.album.is_none()
&& !first_tn
.map(|tn| tn.height == tn.width)
.unwrap_or_default(),
),
},
d.video_id,
)
item.playlist_item_data.map(|d| MusicPage {
id: d.video_id,
typ: MusicPageType::Track {
vtype: MusicVideoType::from_is_video(
self.album.is_none()
&& !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(),
),
},
})
})
.or_else(|| {
first_tn.and_then(|tn| {
util::video_id_from_thumbnail_url(&tn.url).map(|id| {
(
MusicPageType::Track {
vtype: MusicVideoType::from_is_video(
self.album.is_none() && tn.width != tn.height,
),
},
id,
)
util::video_id_from_thumbnail_url(&tn.url).map(|id| MusicPage {
id,
typ: MusicPageType::Track {
vtype: MusicVideoType::from_is_video(
self.album.is_none() && tn.width != tn.height,
),
},
})
})
});
match pt_id {
match music_page.map(|mp| (mp.typ, mp.id)) {
// Track
Some((MusicPageType::Track { vtype }, id)) => {
let title = title.ok_or_else(|| format!("track {id}: could not get title"))?;
@ -852,10 +834,6 @@ impl MusicListMapper {
}
// Tracks were already handled above
MusicPageType::Track { .. } => unreachable!(),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
}
}
}
None => {
@ -875,12 +853,12 @@ impl MusicListMapper {
let subtitle_p2 = subtitle_parts.next();
match item.navigation_endpoint.music_page() {
Some((page_type, id)) => match page_type {
Some(music_page) => match music_page.typ {
MusicPageType::Track { vtype } => {
let (artists, by_va) = map_artists(subtitle_p1);
self.items.push(MusicItem::Track(TrackItem {
id,
id: music_page.id,
name: item.title,
duration: None,
cover: item.thumbnail_renderer.into(),
@ -910,7 +888,7 @@ impl MusicListMapper {
});
self.items.push(MusicItem::Artist(ArtistItem {
id,
id: music_page.id,
name: item.title,
avatar: item.thumbnail_renderer.into(),
subscriber_count,
@ -947,12 +925,15 @@ impl MusicListMapper {
(Vec::new(), true)
}
_ => {
return Err(format!("could not parse subtitle of album {id}"));
return Err(format!(
"could not parse subtitle of album {}",
music_page.id
));
}
};
self.items.push(MusicItem::Album(AlbumItem {
id,
id: music_page.id,
name: item.title,
cover: item.thumbnail_renderer.into(),
artist_id: artists.first().and_then(|a| a.id.clone()),
@ -974,7 +955,7 @@ impl MusicListMapper {
.and_then(|p| p.0.into_iter().find_map(|c| ChannelId::try_from(c).ok()));
self.items.push(MusicItem::Playlist(MusicPlaylistItem {
id,
id: music_page.id,
name: item.title,
thumbnail: item.thumbnail_renderer.into(),
channel,
@ -984,10 +965,6 @@ impl MusicListMapper {
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => Ok(None),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
}
},
None => Err("could not determine item type".to_owned()),
}
@ -1009,7 +986,7 @@ impl MusicListMapper {
let subtitle_p4 = subtitle_parts.next();
let item_type = match card.on_tap.music_page() {
Some((page_type, id)) => match page_type {
Some(music_page) => match music_page.typ {
MusicPageType::Artist => {
let subscriber_count = subtitle_p2.and_then(|p| {
util::parse_large_numstr_or_warn(
@ -1020,7 +997,7 @@ impl MusicListMapper {
});
self.items.push(MusicItem::Artist(ArtistItem {
id,
id: music_page.id,
name: card.title,
avatar: card.thumbnail.into(),
subscriber_count,
@ -1034,7 +1011,7 @@ impl MusicListMapper {
.unwrap_or_default();
self.items.push(MusicItem::Album(AlbumItem {
id,
id: music_page.id,
name: card.title,
cover: card.thumbnail.into(),
artist_id: artists.first().and_then(|a| a.id.clone()),
@ -1050,7 +1027,7 @@ impl MusicListMapper {
let (artists, by_va) = map_artists(subtitle_p3);
self.items.push(MusicItem::Track(TrackItem {
id,
id: music_page.id,
name: card.title,
duration: None,
cover: card.thumbnail.into(),
@ -1087,7 +1064,7 @@ impl MusicListMapper {
};
self.items.push(MusicItem::Track(TrackItem {
id,
id: music_page.id,
name: card.title,
duration,
cover: card.thumbnail.into(),
@ -1113,7 +1090,7 @@ impl MusicListMapper {
subtitle_p3.and_then(|p| util::parse_numeric(p.first_str()).ok());
self.items.push(MusicItem::Playlist(MusicPlaylistItem {
id,
id: music_page.id,
name: card.title,
thumbnail: card.thumbnail.into(),
channel,
@ -1123,10 +1100,6 @@ impl MusicListMapper {
Some(MusicItemType::Playlist)
}
MusicPageType::None => None,
MusicPageType::Unknown => {
self.has_unknown = true;
None
}
},
None => {
self.warnings
@ -1201,20 +1174,6 @@ impl MusicListMapper {
warnings: self.warnings,
}
}
/// Sometimes the YT Music API returns responses containing unknown items.
///
/// In this case, the response data is likely missing some fields, which leads to
/// parsing errors and wrong data being extracted.
///
/// Therefore it is safest to discard such responses and retry the request.
pub fn check_unknown(&self) -> Result<(), ExtractionError> {
if self.has_unknown {
Err(ExtractionError::InvalidData("unknown YTM items".into()))
} else {
Ok(())
}
}
}
/// Map TextComponents containing artist names to a list of artists and a 'Various Artists' flag

63
src/client/response/url_endpoint.rs
View file

@ -1,7 +1,7 @@
use serde::Deserialize;
use serde_with::{serde_as, DefaultOnError};
use crate::model::UrlTarget;
use crate::{model::UrlTarget, util};
/// navigation/resolve_url response model
#[derive(Debug, Deserialize)]
@ -185,6 +185,10 @@ pub(crate) enum PageType {
Channel,
#[serde(rename = "MUSIC_PAGE_TYPE_PLAYLIST", alias = "WEB_PAGE_TYPE_PLAYLIST")]
Playlist,
#[serde(rename = "MUSIC_PAGE_TYPE_PODCAST_SHOW_DETAIL_PAGE")]
Podcast,
#[serde(rename = "MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE")]
Episode,
#[default]
Unknown,
}
@ -195,6 +199,13 @@ impl PageType {
PageType::Artist | PageType::Channel => Some(UrlTarget::Channel { id }),
PageType::Album => Some(UrlTarget::Album { id }),
PageType::Playlist => Some(UrlTarget::Playlist { id }),
PageType::Podcast => Some(UrlTarget::Playlist {
id: util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX),
}),
PageType::Episode => Some(UrlTarget::Video {
id: util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX),
start_time: 0,
}),
PageType::Unknown => None,
}
}
@ -206,7 +217,6 @@ pub(crate) enum MusicPageType {
Album,
Playlist,
Track { vtype: MusicVideoType },
Unknown,
None,
}
@ -215,16 +225,40 @@ impl From<PageType> for MusicPageType {
match t {
PageType::Artist => MusicPageType::Artist,
PageType::Album => MusicPageType::Album,
PageType::Playlist => MusicPageType::Playlist,
PageType::Channel => MusicPageType::None,
PageType::Unknown => MusicPageType::Unknown,
PageType::Playlist | PageType::Podcast => MusicPageType::Playlist,
PageType::Channel | PageType::Unknown => MusicPageType::None,
PageType::Episode => MusicPageType::Track {
vtype: MusicVideoType::Episode,
},
}
}
}
pub(crate) struct MusicPage {
pub id: String,
pub typ: MusicPageType,
}
impl MusicPage {
/// Create a new MusicPage object, applying the required ID fixes when
/// mapping a browse link
pub fn from_browse(mut id: String, typ: PageType) -> Self {
if typ == PageType::Podcast {
id = util::strip_prefix(&id, util::PODCAST_PLAYLIST_PREFIX);
} else if typ == PageType::Episode && id.len() == 15 {
id = util::strip_prefix(&id, util::PODCAST_EPISODE_PREFIX);
}
Self {
id,
typ: typ.into(),
}
}
}
impl NavigationEndpoint {
/// Get the YouTube Music page and id from a browse/watch endpoint
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
pub(crate) fn music_page(self) -> Option<MusicPage> {
match self {
NavigationEndpoint::Watch { watch_endpoint } => {
if watch_endpoint
@ -233,17 +267,20 @@ impl NavigationEndpoint {
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
Some((MusicPageType::None, watch_endpoint.video_id))
Some(MusicPage {
id: watch_endpoint.video_id,
typ: MusicPageType::None,
})
} else {
Some((
MusicPageType::Track {
Some(MusicPage {
id: watch_endpoint.video_id,
typ: MusicPageType::Track {
vtype: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type,
},
watch_endpoint.video_id,
))
})
}
}
NavigationEndpoint::Browse {
@ -251,9 +288,9 @@ impl NavigationEndpoint {
} => browse_endpoint
.browse_endpoint_context_supported_configs
.map(|config| {
(
config.browse_endpoint_context_music_config.page_type.into(),
MusicPage::from_browse(
browse_endpoint.browse_id,
config.browse_endpoint_context_music_config.page_type,
)
}),
NavigationEndpoint::Url { .. } => None,

21
src/serializer/text.rs
View file

@ -6,7 +6,9 @@ use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs, VecSkipError};
use crate::{
client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType},
client::response::url_endpoint::{
MusicPage, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
},
model::UrlTarget,
util,
};
@ -419,6 +421,23 @@ impl TextComponent {
| TextComponent::Text { text } => text,
}
}
pub fn music_page(self) -> Option<MusicPage> {
match self {
TextComponent::Video {
video_id, vtype, ..
} => Some(MusicPage {
id: video_id,
typ: MusicPageType::Track { vtype },
}),
TextComponent::Browse {
page_type,
browse_id,
..
} => Some(MusicPage::from_browse(browse_id, page_type)),
_ => None,
}
}
}
impl From<TextComponent> for String {

7
src/util/mod.rs
View file

@ -41,6 +41,8 @@ pub const DOT_SEPARATOR: &str = " • ";
pub const VARIOUS_ARTISTS: &str = "Various Artists";
pub const PLAYLIST_ID_ALBUM_PREFIX: &str = "OLAK";
pub const ARTIST_DISCOGRAPHY_PREFIX: &str = "MPAD";
pub const PODCAST_PLAYLIST_PREFIX: &str = "MPSP";
pub const PODCAST_EPISODE_PREFIX: &str = "MPED";
const CONTENT_PLAYBACK_NONCE_ALPHABET: &[u8; 64] =
b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@ -474,6 +476,11 @@ pub fn country_from_name(name: &str) -> Option<Country> {
.map(|i| COUNTRIES[i])
}
/// Strip prefix from string if presend
pub fn strip_prefix(s: &str, prefix: &str) -> String {
s.strip_prefix(prefix).unwrap_or(s).to_string()
}
/// An iterator over the chars in a string (in str format)
pub struct SplitChar<'a> {
txt: &'a str,

36
tests/youtube.rs
View file

@ -1664,7 +1664,9 @@ fn music_search_tracks(rp: RustyPipe, unlocalized: bool) {
.items
.iter()
.find(|a| a.id == "BL-aIpCLWnU")
.unwrap();
.unwrap_or_else(|| {
panic!("could not find track, got {:#?}", &res.items.items);
});
assert_eq!(track.name, "Black Mamba");
assert!(!track.cover.is_empty(), "got no cover");
@ -1699,7 +1701,9 @@ fn music_search_videos(rp: RustyPipe, unlocalized: bool) {
.items
.iter()
.find(|a| a.id == "ZeerrnuLi5E")
.unwrap();
.unwrap_or_else(|| {
panic!("could not find video, got {:#?}", &res.items.items);
});
assert_eq!(track.name, "Black Mamba");
assert!(!track.cover.is_empty(), "got no cover");
@ -1739,7 +1743,12 @@ fn music_search_episode(rp: RustyPipe, #[case] videos: bool) {
.tracks
};
let track = &tracks.iter().find(|a| a.id == "Zq_-LDy7AgE").unwrap();
let track = &tracks
.iter()
.find(|a| a.id == "Zq_-LDy7AgE")
.unwrap_or_else(|| {
panic!("could not find episode, got {:#?}", &tracks);
});
assert_eq!(track.artists.len(), 1);
let track_artist = &track.artists[0];
@ -1805,7 +1814,14 @@ fn music_search_albums(
) {
let res = tokio_test::block_on(rp.query().music_search_albums(query)).unwrap();
let album = &res.items.items.iter().find(|a| a.id == id).unwrap();
let album = &res
.items
.items
.iter()
.find(|a| a.id == id)
.unwrap_or_else(|| {
panic!("could not find album, got {:#?}", &res.items.items);
});
assert_eq!(album.name, name);
assert_eq!(album.artists.len(), 1);
@ -1836,7 +1852,9 @@ fn music_search_artists(rp: RustyPipe, unlocalized: bool) {
.items
.iter()
.find(|a| a.id == "UCIh4j8fXWf2U0ro0qnGU8Mg")
.unwrap();
.unwrap_or_else(|| {
panic!("could not find artist, got {:#?}", &res.items.items);
});
if unlocalized {
assert_eq!(artist.name, "Namika");
}
@ -1871,7 +1889,9 @@ fn music_search_playlists(rp: RustyPipe, unlocalized: bool) {
.items
.iter()
.find(|p| p.id == "RDCLAK5uy_nLtxizvEMkzYQUrA-bFf6MnBeR4bGYWUQ")
.expect("no playlist");
.unwrap_or_else(|| {
panic!("could not find playlist, got {:#?}", &res.items.items);
});
if unlocalized {
assert_eq!(playlist.name, "Today's Rock Hits");
@ -1901,7 +1921,9 @@ fn music_search_playlists_community(rp: RustyPipe) {
.items
.iter()
.find(|p| p.id == "PLMC9KNkIncKtGvr2kFRuXBVmBev6cAJ2u")
.expect("no playlist");
.unwrap_or_else(|| {
panic!("could not find playlist, got {:#?}", &res.items.items);
});
assert_eq!(
playlist.name,