Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: a/b test 8: parsing view count for tracks

Browse source
This commit is contained in:
ThetaDev 2023-08-23 11:28:00 +02:00
parent 93e5ad22e9
commit 22e298ff98
33 changed files with 389 additions and 260 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/client/music_search.rs
View file

@ -339,7 +339,7 @@ impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchSuggestion>, ExtractionError> {
let mut mapper = MusicListMapper::new(lang);
let mut mapper = MusicListMapper::new_search_suggest(lang);
let mut terms = Vec::new();
for section in self.contents {

277
src/client/response/music_item.rs
View file

@ -16,7 +16,9 @@ use crate::{
};
use super::{
url_endpoint::{BrowseEndpointWrap, MusicPageType, NavigationEndpoint, PageType},
url_endpoint::{
BrowseEndpointWrap, MusicPageType, MusicVideoType, NavigationEndpoint, PageType,
},
ContentsRenderer, MusicContinuationData, Thumbnails, ThumbnailsWrap,
};
@ -429,6 +431,7 @@ pub(crate) struct MusicListMapper {
artists: Option<(Vec<ArtistId>, bool)>,
album: Option<AlbumId>,
artist_page: bool,
search_suggestion: bool,
items: Vec<MusicItem>,
warnings: Vec<String>,
/// True if unknown items were mapped
@ -450,6 +453,20 @@ impl MusicListMapper {
artists: None,
album: None,
artist_page: false,
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
}
}
pub fn new_search_suggest(lang: Language) -> Self {
Self {
lang,
artists: None,
album: None,
artist_page: false,
search_suggestion: true,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
@ -463,6 +480,7 @@ impl MusicListMapper {
artists: Some((vec![artist], false)),
album: None,
artist_page: true,
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
@ -476,6 +494,7 @@ impl MusicListMapper {
artists: Some((artists, by_va)),
album: Some(album),
artist_page: false,
search_suggestion: false,
items: Vec::new(),
warnings: Vec::new(),
has_unknown: false,
@ -515,6 +534,7 @@ impl MusicListMapper {
let c1 = columns.next();
let c2 = columns.next();
let c3 = columns.next();
let c4 = columns.next();
let title = c1.as_ref().map(|col| col.renderer.text.to_string());
@ -532,10 +552,8 @@ impl MusicListMapper {
c1.and_then(|c1| {
c1.renderer.text.0.into_iter().next().and_then(|t| match t {
crate::serializer::text::TextComponent::Video {
video_id,
is_video,
..
} => Some((MusicPageType::Track { is_video }, video_id)),
video_id, vtype, ..
} => Some((MusicPageType::Track { vtype }, video_id)),
crate::serializer::text::TextComponent::Browse {
page_type,
browse_id,
@ -549,8 +567,12 @@ impl MusicListMapper {
item.playlist_item_data.map(|d| {
(
MusicPageType::Track {
is_video: self.album.is_none()
&& !first_tn.map(|tn| tn.height == tn.width).unwrap_or_default(),
vtype: MusicVideoType::from_is_video(
self.album.is_none()
&& !first_tn
.map(|tn| tn.height == tn.width)
.unwrap_or_default(),
),
},
d.video_id,
)
@ -561,7 +583,9 @@ impl MusicListMapper {
util::video_id_from_thumbnail_url(&tn.url).map(|id| {
(
MusicPageType::Track {
is_video: self.album.is_none() && tn.width != tn.height,
vtype: MusicVideoType::from_is_video(
self.album.is_none() && tn.width != tn.height,
),
},
id,
)
@ -571,19 +595,28 @@ impl MusicListMapper {
match pt_id {
// Track
Some((MusicPageType::Track { is_video }, id)) => {
Some((MusicPageType::Track { vtype }, id)) => {
let title = title.ok_or_else(|| format!("track {id}: could not get title"))?;
let (artists_p, album_p, duration_p) = match item.flex_column_display_style {
#[derive(Default)]
struct Parsed {
artists: Option<TextComponents>,
album: Option<TextComponents>,
duration: Option<TextComponents>,
view_count: Option<TextComponents>,
}
let p = match item.flex_column_display_style {
// Search result
FlexColumnDisplayStyle::TwoLines => {
// Is this a related track?
if !is_video && item.item_height == ItemHeight::Compact {
(
c2.map(TextComponents::from),
c3.map(TextComponents::from),
None,
)
// Is this a related track (from the "similar titles" tab in the player)?
if vtype != MusicVideoType::Video && item.item_height == ItemHeight::Compact
{
Parsed {
artists: c2.map(TextComponents::from),
album: c3.map(TextComponents::from),
..Default::default()
}
} else {
let mut subtitle_parts = c2
.ok_or_else(|| format!("track {id}: could not get subtitle"))?
@ -594,62 +627,98 @@ impl MusicListMapper {
// Is this a related video?
if item.item_height == ItemHeight::Compact {
(subtitle_parts.next(), subtitle_parts.next(), None)
Parsed {
artists: subtitle_parts.next(),
view_count: subtitle_parts.next(),
..Default::default()
}
}
// Is this an item from search suggestion?
else if self.search_suggestion {
// Skip first part (track type)
subtitle_parts.next();
Parsed {
artists: subtitle_parts.next(),
album: c3.map(TextComponents::from),
view_count: subtitle_parts.next(),
..Default::default()
}
}
// Is it a podcast episode?
else if subtitle_parts.len() <= 3 && c3.is_some() {
(subtitle_parts.next_back(), None, None)
else if vtype == MusicVideoType::Episode {
Parsed {
artists: subtitle_parts.next_back(),
..Default::default()
}
} else {
// Skip first part (track type)
if subtitle_parts.len() > 3
|| (is_video && subtitle_parts.len() == 2)
|| (vtype == MusicVideoType::Video && subtitle_parts.len() == 2)
{
subtitle_parts.next();
}
(
subtitle_parts.next(),
subtitle_parts.next(),
subtitle_parts.next(),
)
match vtype {
MusicVideoType::Video => Parsed {
artists: subtitle_parts.next(),
view_count: subtitle_parts.next(),
duration: subtitle_parts.next(),
..Default::default()
},
_ => Parsed {
artists: subtitle_parts.next(),
album: subtitle_parts.next(),
duration: subtitle_parts.next(),
view_count: c3.map(TextComponents::from),
},
}
}
}
}
// Playlist item
FlexColumnDisplayStyle::Default => (
c2.map(TextComponents::from),
c3.map(TextComponents::from),
item.fixed_columns
FlexColumnDisplayStyle::Default => {
let artists = c2.map(TextComponents::from);
let duration = item
.fixed_columns
.into_iter()
.next()
.map(TextComponents::from),
),
.map(TextComponents::from);
if self.album.is_some() {
Parsed {
artists,
view_count: c3.map(TextComponents::from),
duration,
..Default::default()
}
} else if self.artist_page && c4.is_some() {
Parsed {
artists,
view_count: c3.map(TextComponents::from),
album: c4.map(TextComponents::from),
duration,
}
} else {
Parsed {
artists,
album: c3.map(TextComponents::from),
duration,
..Default::default()
}
}
}
};
let duration = duration_p.and_then(|p| util::parse_video_length(p.first_str()));
let (album, view_count) = match (item.flex_column_display_style, is_video) {
// The album field contains the view count for search videos
(FlexColumnDisplayStyle::TwoLines, true) => (
None,
album_p.and_then(|p| {
util::parse_large_numstr_or_warn(
p.first_str(),
self.lang,
&mut self.warnings,
)
}),
),
(_, false) => (
album_p
.and_then(|p| p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok())),
None,
),
(FlexColumnDisplayStyle::Default, true) => (None, None),
};
let album = album.or_else(|| self.album.clone());
let (mut artists, by_va) = map_artists(artists_p);
let duration = p
.duration
.and_then(|p| util::parse_video_length(p.first_str()));
let album = p
.album
.and_then(|p| p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok()))
.or_else(|| self.album.clone());
let view_count = p.view_count.and_then(|p| {
util::parse_large_numstr_or_warn(p.first_str(), self.lang, &mut self.warnings)
});
let (mut artists, by_va) = map_artists(p.artists);
// Extract artist id from dropdown menu
let artist_id = map_artist_id_fallback(item.menu, artists.first());
@ -685,7 +754,7 @@ impl MusicListMapper {
artist_id,
album,
view_count,
is_video,
is_video: vtype.is_video(),
track_nr,
by_va,
}));
@ -807,7 +876,7 @@ impl MusicListMapper {
match item.navigation_endpoint.music_page() {
Some((page_type, id)) => match page_type {
MusicPageType::Track { is_video } => {
MusicPageType::Track { vtype } => {
let (artists, by_va) = map_artists(subtitle_p1);
self.items.push(MusicItem::Track(TrackItem {
@ -825,7 +894,7 @@ impl MusicListMapper {
&mut self.warnings,
)
}),
is_video,
is_video: vtype.is_video(),
track_nr: None,
by_va,
}));
@ -976,43 +1045,61 @@ impl MusicListMapper {
}));
Some(MusicItemType::Album)
}
MusicPageType::Track { is_video } => {
let (artists, by_va) = map_artists(subtitle_p2);
let duration =
subtitle_p4.and_then(|p| util::parse_video_length(p.first_str()));
let (album, view_count) = if is_video {
(
None,
subtitle_p3.and_then(|p| {
util::parse_large_numstr_or_warn(
p.first_str(),
self.lang,
&mut self.warnings,
)
}),
)
} else {
(
subtitle_p3.and_then(|p| {
p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok())
}),
None,
)
};
MusicPageType::Track { vtype } => {
if vtype == MusicVideoType::Episode {
let (artists, by_va) = map_artists(subtitle_p3);
self.items.push(MusicItem::Track(TrackItem {
id,
name: card.title,
duration,
cover: card.thumbnail.into(),
artist_id: artists.first().and_then(|a| a.id.clone()),
artists,
album,
view_count,
is_video,
track_nr: None,
by_va,
}));
self.items.push(MusicItem::Track(TrackItem {
id,
name: card.title,
duration: None,
cover: card.thumbnail.into(),
artist_id: artists.first().and_then(|a| a.id.clone()),
artists,
album: None,
view_count: None,
is_video: vtype.is_video(),
track_nr: None,
by_va,
}));
} else {
let (artists, by_va) = map_artists(subtitle_p2);
let duration =
subtitle_p4.and_then(|p| util::parse_video_length(p.first_str()));
let (album, view_count) = if vtype.is_video() {
(
None,
subtitle_p3.and_then(|p| {
util::parse_large_numstr_or_warn(
p.first_str(),
self.lang,
&mut self.warnings,
)
}),
)
} else {
(
subtitle_p3.and_then(|p| {
p.0.into_iter().find_map(|c| AlbumId::try_from(c).ok())
}),
None,
)
};
self.items.push(MusicItem::Track(TrackItem {
id,
name: card.title,
duration,
cover: card.thumbnail.into(),
artist_id: artists.first().and_then(|a| a.id.clone()),
artists,
album,
view_count,
is_video: vtype.is_video(),
track_nr: None,
by_va,
}));
}
Some(MusicItemType::Track)
}
MusicPageType::Playlist => {

23
src/client/response/url_endpoint.rs
View file

@ -151,6 +151,22 @@ pub(crate) enum MusicVideoType {
Video,
#[serde(rename = "MUSIC_VIDEO_TYPE_ATV")]
Track,
#[serde(rename = "MUSIC_VIDEO_TYPE_PODCAST_EPISODE")]
Episode,
}
impl MusicVideoType {
pub fn is_video(self) -> bool {
self != Self::Track
}
pub fn from_is_video(is_video: bool) -> Self {
if is_video {
Self::Video
} else {
Self::Track
}
}
}
#[derive(Default, Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
@ -189,7 +205,7 @@ pub(crate) enum MusicPageType {
Artist,
Album,
Playlist,
Track { is_video: bool },
Track { vtype: MusicVideoType },
Unknown,
None,
}
@ -221,11 +237,10 @@ impl NavigationEndpoint {
} else {
Some((
MusicPageType::Track {
is_video: watch_endpoint
vtype: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
.music_video_type,
},
watch_endpoint.video_id,
))

5
src/client/snapshots/rustypipe__client__music_search__tests__map_music_search_suggestion_default.snap
View file

@ -67,7 +67,10 @@ MusicSearchSuggestion(
),
],
artist_id: Some("UC56hLMPuEsERdmTBbR_JGHA"),
album: None,
album: Some(AlbumId(
id: "MPREb_kz546sNB1mH",
name: "Was Spaß macht...",
)),
view_count: None,
is_video: false,
track_nr: None,

9
src/deobfuscate.rs
View file

@ -142,14 +142,15 @@ fn get_sig_fn(player_js: &str) -> Result<String, DeobfError> {
let function_pattern = Regex::new(&function_pattern_str)
.map_err(|_| DeobfError::Other("could not parse function pattern regex"))?;
let deobfuscate_function = "var ".to_owned()
+ function_pattern
let deobfuscate_function = format!(
"var {};",
function_pattern
.captures(player_js)
.ok_or(DeobfError::Extraction("deobf function"))?
.get(1)
.unwrap()
.as_str()
+ ";";
);
static HELPER_OBJECT_NAME_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#";([A-Za-z0-9_\$]{2,3})\...\("#).unwrap());
@ -203,7 +204,7 @@ fn get_nsig_fn_name(player_js: &str) -> Result<String, DeobfError> {
.as_str()
.parse::<usize>()
.or(Err(DeobfError::Other("could not parse array_num")))?;
let array_pattern_str = format!(r#"var {}\s*=\s*\[(.+?)][;,]"#, regex::escape(function_name));
let array_pattern_str = format!(r#"var {}\s*=\s*\[(.+?)]"#, regex::escape(function_name));
let array_pattern = Regex::new(&array_pattern_str).or(Err(DeobfError::Other(
"could not parse helper pattern regex",
)))?;

3
src/model/richtext.rs
View file

@ -170,6 +170,7 @@ mod tests {
use once_cell::sync::Lazy;
use crate::client::response::url_endpoint::MusicVideoType;
use crate::serializer::text;
static TEXT_SOURCE: Lazy<text::TextComponents> = Lazy::new(|| {
@ -177,7 +178,7 @@ mod tests {
text::TextComponent::Text { text: "🎧Listen and download aespa's debut single \"Black Mamba\": ".to_owned() },
text::TextComponent::Web { text: "https://smarturl.it/aespa_BlackMamba".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFY1QmpQamJPSms0Z1FnVTlQUS00ZFhBZnBJZ3xBQ3Jtc0tuRGJBanludGoyRnphb2dZWVd3cUNnS3dEd0FnNHFOZEY1NHBJaHFmLXpaWUJwX3ZucDZxVnpGeHNGX1FpMzFkZW9jQkI2Mi1wNGJ1UVFNN3h1MnN3R3JLMzdxU01nZ01POHBGcmxHU2puSUk1WHRzQQ&q=https%3A%2F%2Fsmarturl.it%2Faespa_BlackMamba&v=ZeerrnuLi5E".to_owned() },
text::TextComponent::Text { text: "\n🐍The Debut Stage ".to_owned() },
text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0, is_video: true },
text::TextComponent::Video { text: "https://youtu.be/Ky5RT5oGg0w".to_owned(), video_id: "Ky5RT5oGg0w".to_owned(), start_time: 0, vtype: MusicVideoType::Video },
text::TextComponent::Text { text: "\n\n🎟️ aespa Showcase SYNK in LA! Tickets now on sale: ".to_owned() },
text::TextComponent::Web { text: "https://www.ticketmaster.com/event/0A...".to_owned(), url: "https://www.youtube.com/redirect?event=video_description&redir_token=QUFFLUhqbFpUMEZiaXJWWkszaVZXaEM0emxWU1JQV3NoQXxBQ3Jtc0tuU2g4VWNPNE5UY3hoSWYtamFzX0h4bUVQLVJiRy1ubDZrTnh3MUpGdDNSaUo0ZlMyT3lUM28ycUVBdHJLMndGcDhla3BkOFpxSVFfOS1QdVJPVHBUTEV1LXpOV0J2QXdhV05lV210cEJtZUJMeHdaTQ&q=https%3A%2F%2Fwww.ticketmaster.com%2Fevent%2F0A005CCD9E871F6E&v=ZeerrnuLi5E".to_owned() },
text::TextComponent::Text { text: "\n\nSubscribe to aespa Official YouTube Channel!\n".to_owned() },

2
src/serializer/snapshots/rustypipe__serializer__text__tests__t_attributed_description.snap
View file

@ -19,7 +19,7 @@ SAttributed {
text: "aespa 에스파 'Black ...",
video_id: "Ky5RT5oGg0w",
start_time: 0,
is_video: true,
vtype: Video,
},
Text {
text: "\n\n🎟\u{fe0f} aespa Showcase SYNK in LA! Tickets now on sale: ",

10
src/serializer/text.rs
View file

@ -94,8 +94,7 @@ pub(crate) enum TextComponent {
text: String,
video_id: String,
start_time: u32,
/// True if the item is a video, false if it is a YTM track
is_video: bool,
vtype: MusicVideoType,
},
Browse {
text: String,
@ -167,11 +166,10 @@ fn map_text_component(text: String, nav: Option<NavigationEndpoint>) -> TextComp
text,
video_id: watch_endpoint.video_id,
start_time: watch_endpoint.start_time_seconds,
is_video: watch_endpoint
vtype: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
.music_video_type,
},
Some(NavigationEndpoint::Browse {
browse_endpoint,
@ -612,7 +610,7 @@ mod tests {
text: "DEEP",
video_id: "wZIoIgz5mbs",
start_time: 0,
is_video: true,
vtype: Video,
},
}
"###);