fix: fetch artist albums continuation

This commit is contained in:
ThetaDev 2024-10-22 23:56:59 +02:00
parent be18d89ea6
commit b589061a40
No known key found for this signature in database
GPG key ID: E319D3C5148D65B6
17 changed files with 50309 additions and 38906 deletions

View file

@ -38,6 +38,7 @@ use crate::{
cache::{CacheStorage, FileStorage, DEFAULT_CACHE_FILE},
deobfuscate::DeobfData,
error::{Error, ExtractionError},
model::ArtistId,
param::{Country, Language},
report::{FileReporter, Level, Report, Reporter, RustyPipeInfo, DEFAULT_REPORT_DIR},
serializer::MapResult,
@ -1415,7 +1416,7 @@ impl RustyPipeQuery {
}
};
tracing::debug!("mapped response");
tracing::trace!("mapped response");
Ok(RequestResult { res, status, body })
}
@ -1468,10 +1469,9 @@ impl RustyPipeQuery {
/// - `method`: HTTP method
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
/// - `body`: Serializable request body to be sent in json format
/// - `visitor_data`: YouTube visitor data cookie
/// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs).
/// - `ctx_src`: Context source (additional parameters for fetching and mapping, used to build the MapRespCtx)
#[allow(clippy::too_many_arguments)]
async fn execute_request_deobf<
async fn execute_request_ctx<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
B: Serialize + ?Sized,
@ -1482,25 +1482,25 @@ impl RustyPipeQuery {
id: &str,
endpoint: &str,
body: &B,
visitor_data: Option<&str>,
deobf: Option<&DeobfData>,
ctx_src: MapRespCtxSource<'_>,
) -> Result<M, Error> {
tracing::debug!("getting {}({})", operation, id);
let request = self
.request_builder(ctype, endpoint, visitor_data)
.await
.json(body)
.build()?;
let ctx = MapRespCtx {
id,
lang: self.opts.lang,
deobf,
visitor_data: visitor_data.or(self.opts.visitor_data.as_deref()),
deobf: ctx_src.deobf,
visitor_data: ctx_src.visitor_data.or(self.opts.visitor_data.as_deref()),
client_type: ctype,
artist: ctx_src.artist,
};
let request = self
.request_builder(ctype, endpoint, ctx.visitor_data)
.await
.json(body)
.build()?;
let req_res = self.yt_request::<R, M>(&request, &ctx).await?;
// Uncomment to debug response text
@ -1533,7 +1533,7 @@ impl RustyPipeQuery {
operation: &format!("{operation}({id})"),
error,
msgs,
deobf_data: deobf.cloned(),
deobf_data: ctx.deobf.cloned(),
http_request: crate::report::HTTPRequest {
url: request.url().as_str(),
method: request.method().as_str(),
@ -1587,10 +1587,18 @@ impl RustyPipeQuery {
endpoint: &str,
body: &B,
) -> Result<M, Error> {
self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None, None)
.await
self.execute_request_ctx::<R, M, B>(
ctype,
operation,
id,
endpoint,
body,
MapRespCtxSource::default(),
)
.await
}
/*
/// Execute a request to the YouTube API, then map the response.
///
/// Creates a report in case of failure for easy debugging.
@ -1629,6 +1637,7 @@ impl RustyPipeQuery {
)
.await
}
*/
/// Execute a request to the YouTube API and return the response string
///
@ -1664,6 +1673,14 @@ struct MapRespCtx<'a> {
deobf: Option<&'a DeobfData>,
visitor_data: Option<&'a str>,
client_type: ClientType,
artist: Option<ArtistId>,
}
#[derive(Default)]
struct MapRespCtxSource<'a> {
visitor_data: Option<&'a str>,
deobf: Option<&'a DeobfData>,
artist: Option<ArtistId>,
}
impl<'a> MapRespCtx<'a> {
@ -1676,6 +1693,16 @@ impl<'a> MapRespCtx<'a> {
deobf: None,
visitor_data: None,
client_type: ClientType::Desktop,
artist: None,
}
}
}
impl<'a> MapRespCtxSource<'a> {
fn visitor_data(visitor_data: &'a str) -> Self {
Self {
visitor_data: Some(visitor_data),
..Default::default()
}
}
}

View file

@ -5,16 +5,19 @@ use regex::Regex;
use tracing::debug;
use crate::{
client::response::url_endpoint::NavigationEndpoint,
client::{response::url_endpoint::NavigationEndpoint, MapRespCtxSource, QContinuation},
error::{Error, ExtractionError},
model::{AlbumItem, ArtistId, MusicArtist},
model::{
paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistId, MusicArtist, MusicItem,
},
param::{AlbumFilter, AlbumOrder},
serializer::MapResult,
util,
util::{self, ProtoBuilder},
};
use super::{
response::{self, music_item::MusicListMapper, url_endpoint::PageType},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
};
impl RustyPipeQuery {
@ -56,7 +59,9 @@ impl RustyPipeQuery {
.await?;
if can_fetch_more {
artist.albums = self.music_artist_albums(artist_id).await?;
artist.albums = self
.music_artist_albums(artist_id, None, Some(AlbumOrder::Recency))
.await?;
}
Ok(artist)
@ -73,21 +78,62 @@ impl RustyPipeQuery {
}
/// Get a list of all albums of a YouTube Music artist
pub async fn music_artist_albums(&self, artist_id: &str) -> Result<Vec<AlbumItem>, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
pub async fn music_artist_albums(
&self,
artist_id: &str,
filter: Option<AlbumFilter>,
order: Option<AlbumOrder>,
) -> Result<Vec<AlbumItem>, Error> {
let visitor_data = self.get_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QBrowseParams {
context: context.clone(),
browse_id: &format!("{}{}", util::ARTIST_DISCOGRAPHY_PREFIX, artist_id),
params: &albums_param(filter, order),
};
self.execute_request::<response::MusicArtistAlbums, _, _>(
ClientType::DesktopMusic,
"music_artist_albums",
artist_id,
"browse",
&request_body,
)
.await
let first_page = self
.execute_request_ctx::<response::MusicArtistAlbums, _, _>(
ClientType::DesktopMusic,
"music_artist_albums",
artist_id,
"browse",
&request_body,
MapRespCtxSource::visitor_data(&visitor_data),
)
.await?;
let mut albums = first_page.albums;
let mut ctoken = first_page.ctoken;
while let Some(tkn) = &ctoken {
let request_body = QContinuation {
context: context.clone(),
continuation: tkn,
};
let resp: Paginator<MusicItem> = self
.execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
ClientType::DesktopMusic,
"music_artist_albums_cont",
artist_id,
"browse",
&request_body,
MapRespCtxSource {
artist: Some(first_page.artist.clone()),
visitor_data: Some(&visitor_data),
..Default::default()
},
)
.await?;
if resp.items.is_empty() {
tracing::warn!("artist albums [{artist_id}] empty continuation");
}
ctoken = resp.ctoken;
albums.extend(resp.items.into_iter().filter_map(AlbumItem::from_ytm_item));
}
Ok(albums)
}
}
@ -280,11 +326,18 @@ fn map_artist_page(
})
}
impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
#[derive(Debug)]
struct FirstAlbumPage {
albums: Vec<AlbumItem>,
ctoken: Option<String>,
artist: ArtistId,
}
impl MapResponse<FirstAlbumPage> for response::MusicArtistAlbums {
fn map_response(
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> {
) -> Result<MapResult<FirstAlbumPage>, ExtractionError> {
// dbg!(&self);
let Some(header) = self.header else {
@ -306,27 +359,55 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
.section_list_renderer
.contents;
let mut mapper = MusicListMapper::with_artist(
ctx.lang,
ArtistId {
id: Some(ctx.id.to_owned()),
name: header.music_header_renderer.title,
},
);
let artist_id = ArtistId {
id: Some(ctx.id.to_owned()),
name: header.music_header_renderer.title,
};
let mut mapper = MusicListMapper::with_artist(ctx.lang, artist_id.clone());
let mut ctoken = None;
for grid in grids {
mapper.map_response(grid.grid_renderer.items);
if ctoken.is_none() {
ctoken = grid
.grid_renderer
.continuations
.into_iter()
.next()
.map(|g| g.next_continuation_data.continuation);
}
}
let mapped = mapper.group_items();
Ok(MapResult {
c: mapped.c.albums,
c: FirstAlbumPage {
albums: mapped.c.albums,
ctoken,
artist: artist_id,
},
warnings: mapped.warnings,
})
}
}
fn albums_param(filter: Option<AlbumFilter>, order: Option<AlbumOrder>) -> String {
let mut pb_filter = ProtoBuilder::new();
if let Some(filter) = filter {
pb_filter.varint(1, filter as u64);
}
if let Some(order) = order {
pb_filter.varint(2, order as u64);
}
pb_filter.bytes(3, &[1, 2]);
let mut pb_48 = ProtoBuilder::new();
pb_48.embedded(15, pb_filter);
let mut pb_3 = ProtoBuilder::new();
pb_3.embedded(48, pb_48);
pb_3.to_base64()
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader};
@ -366,11 +447,12 @@ mod tests {
);
assert_eq!(can_fetch_more, album_page_path.is_some());
// Album overview
if let Some(album_page_path) = album_page_path {
let json_file = File::open(album_page_path).unwrap();
let resp: response::MusicArtistAlbums =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let mut map_res: MapResult<Vec<AlbumItem>> =
let map_res: MapResult<FirstAlbumPage> =
resp.map_response(&MapRespCtx::test(id)).unwrap();
assert!(
@ -378,7 +460,29 @@ mod tests {
"deserialization/mapping warnings: {:?}",
map_res.warnings
);
artist.albums.append(&mut map_res.c);
artist.albums = map_res.c.albums;
// Album overview continuation
for i in 2..10 {
let cont_path =
path!(*TESTFILES / "music_artist" / format!("artist_{name}_{i}.json"));
if !cont_path.is_file() {
break;
}
let json_file = File::open(cont_path).unwrap();
let resp: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> =
resp.map_response(&MapRespCtx::test(id)).unwrap();
assert!(!map_res.c.items.is_empty());
artist.albums.extend(
map_res
.c
.items
.into_iter()
.filter_map(AlbumItem::from_ytm_item),
);
}
}
insta::assert_ron_snapshot!(format!("map_music_artist_{name}"), artist);

View file

@ -16,7 +16,7 @@ use super::{
self,
music_item::{map_queue_item, MusicListMapper},
},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, YTContext,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
@ -134,13 +134,13 @@ impl RustyPipeQuery {
tuner_setting_value: "AUTOMIX_SETTING_NORMAL",
};
self.execute_request_vdata::<response::MusicDetails, _, _>(
self.execute_request_ctx::<response::MusicDetails, _, _>(
ClientType::DesktopMusic,
"music_radio",
radio_id,
"next",
&request_body,
Some(&visitor_data),
MapRespCtxSource::visitor_data(&visitor_data),
)
.await
}

View file

@ -17,7 +17,7 @@ use super::{
self,
music_item::{map_album_type, map_artist_id, map_artists, MusicListMapper},
},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery,
};
impl RustyPipeQuery {
@ -42,13 +42,16 @@ impl RustyPipeQuery {
browse_id: &format!("VL{playlist_id}"),
};
self.execute_request_vdata::<response::MusicPlaylist, _, _>(
self.execute_request_ctx::<response::MusicPlaylist, _, _>(
ClientType::DesktopMusic,
"music_playlist",
playlist_id,
"browse",
&request_body,
visitor_data.as_deref(),
MapRespCtxSource {
visitor_data: visitor_data.as_deref(),
..Default::default()
},
)
.await
}

View file

@ -10,7 +10,9 @@ use crate::model::{
use crate::serializer::MapResult;
use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo};
use super::{response, ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery};
use super::{
response, ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QContinuation, RustyPipeQuery,
};
impl RustyPipeQuery {
/// Get more YouTube items from the given continuation token and endpoint
@ -37,13 +39,13 @@ impl RustyPipeQuery {
};
let p = self
.execute_request_vdata::<response::MusicContinuation, Paginator<MusicItem>, _>(
.execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
ClientType::DesktopMusic,
"music_continuation",
ctoken,
endpoint.as_str(),
&request_body,
Some(&visitor_data),
MapRespCtxSource::visitor_data(&visitor_data),
)
.await?;
@ -138,7 +140,11 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> {
let mut mapper = MusicListMapper::new(ctx.lang);
let mut mapper = if let Some(artist) = &ctx.artist {
MusicListMapper::with_artist(ctx.lang, artist.clone())
} else {
MusicListMapper::new(ctx.lang)
};
let mut continuations = Vec::new();
match self.continuation_contents {
@ -156,7 +162,11 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
mapper.map_response(shelf.contents);
}
_ => {}
response::music_item::ItemSection::GridRenderer(mut grid) => {
mapper.map_response(grid.items);
continuations.append(&mut grid.continuations);
}
response::music_item::ItemSection::None => {}
}
}
}
@ -173,6 +183,10 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
}
});
}
Some(response::music_item::ContinuationContents::GridContinuation(mut grid)) => {
mapper.map_response(grid.items);
continuations.append(&mut grid.continuations);
}
None => {}
}
@ -257,6 +271,19 @@ impl<T: FromYtItem> Paginator<T> {
}
Ok(())
}
/// Extend the items of the paginator until the paginator is exhausted.
pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(&mut self, query: Q) -> Result<(), Error> {
let query = query.as_ref();
loop {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
Ok(())
}
}
impl Paginator<Comment> {
@ -334,6 +361,22 @@ macro_rules! paginator {
}
Ok(())
}
/// Extend the items of the paginator until the paginator is exhausted.
pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
) -> Result<(), Error> {
let query = query.as_ref();
loop {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
Ok(())
}
}
};
}

View file

@ -24,7 +24,7 @@ use super::{
self,
player::{self, Format},
},
ClientType, MapRespCtx, MapResponse, MapResult, RustyPipeQuery, YTContext,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, RustyPipeQuery, YTContext,
DEFAULT_PLAYER_CLIENT_ORDER,
};
@ -162,14 +162,16 @@ impl RustyPipeQuery {
}
};
self.execute_request_deobf::<response::Player, _, _>(
self.execute_request_ctx::<response::Player, _, _>(
client_type,
"player",
video_id,
"player",
&request_body,
None,
Some(&deobf),
MapRespCtxSource {
deobf: Some(&deobf),
..Default::default()
},
)
.await
}
@ -763,6 +765,7 @@ mod tests {
deobf: Some(&DEOBF_DATA),
visitor_data: None,
client_type,
artist: None,
})
.unwrap();

View file

@ -13,7 +13,10 @@ use crate::{
util::{self, dictionary, timeago, TryRemove},
};
use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, QBrowse, RustyPipeQuery};
use super::{
response, ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, QBrowse,
RustyPipeQuery,
};
impl RustyPipeQuery {
/// Get a YouTube playlist
@ -34,13 +37,16 @@ impl RustyPipeQuery {
browse_id: &format!("VL{playlist_id}"),
};
self.execute_request_vdata::<response::Playlist, _, _>(
self.execute_request_ctx::<response::Playlist, _, _>(
ClientType::Desktop,
"playlist",
playlist_id,
"browse",
&request_body,
visitor_data.as_deref(),
MapRespCtxSource {
visitor_data: visitor_data.as_deref(),
..Default::default()
},
)
.await
}

View file

@ -335,6 +335,7 @@ pub(crate) enum ContinuationContents {
MusicShelfContinuation(MusicShelf),
SectionListContinuation(ContentsRenderer<ItemSection>),
PlaylistPanelContinuation(PlaylistPanelRenderer),
GridContinuation(GridRenderer),
}
#[derive(Debug, Deserialize)]
@ -381,11 +382,15 @@ pub(crate) struct Grid {
pub grid_renderer: GridRenderer,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GridRenderer {
pub items: MapResult<Vec<MusicResponseItem>>,
pub header: Option<GridHeader>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub continuations: Vec<MusicContinuationData>,
}
#[derive(Debug, Deserialize)]

View file

@ -23,9 +23,10 @@ pub enum ChannelVideoTab {
}
/// Sort order for channel videos
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChannelOrder {
/// Order videos with the latest upload date first (default)
#[default]
Latest = 1,
/// Order videos with the highest number of views first
Popular = 2,
@ -43,3 +44,23 @@ impl ChannelVideoTab {
}
}
}
/// Sort order for YTM artist albums
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AlbumOrder {
/// Sort albums by release date
Recency = 1,
/// Sort albums by popularity
Popularity = 2,
/// Sort albums by their name
Alphabetical = 3,
}
/// Filter for YTM artist albums
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AlbumFilter {
/// Only show albums
Albums = 1,
/// Only show singles
Singles = 2,
}

View file

@ -204,7 +204,7 @@ impl SearchFilter {
if let Some(sort) = self.sort {
pb.varint(1, sort as u64);
}
if !filters.bytes.is_empty() {
if !filters.is_empty() {
pb.embedded(2, filters);
}
if self.verbatim {

View file

@ -1,7 +1,7 @@
/// [`ProtoBuilder`] is used to construct protobuf messages using a builder pattern
#[derive(Debug, Default)]
pub struct ProtoBuilder {
pub bytes: Vec<u8>,
bytes: Vec<u8>,
}
impl ProtoBuilder {
@ -39,6 +39,11 @@ impl ProtoBuilder {
self._varint(val);
}
/// Returns `true` if the builder contains no data
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
/// Write a varint field
pub fn varint(&mut self, field: u32, val: u64) {
self._field(field, 0);
@ -52,6 +57,13 @@ impl ProtoBuilder {
self.bytes.extend_from_slice(string.as_bytes());
}
/// Write a bytes field
pub fn bytes(&mut self, field: u32, bytes: &[u8]) {
self._field(field, 2);
self._varint(bytes.len() as u64);
self.bytes.extend_from_slice(bytes);
}
/// Write an embedded message
///
/// Requires passing another [`ProtoBuilder`] with the embedded message.

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -5,6 +5,7 @@ use std::fmt::Display;
use std::str::FromStr;
use rstest::{fixture, rstest};
use rustypipe::param::{AlbumOrder, LANGUAGES};
use time::{macros::date, OffsetDateTime};
use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery};
@ -1719,7 +1720,11 @@ async fn music_artist(
// Fetch albums seperately
if name != "no_artist" {
let albums = rp.query().music_artist_albums(id).await.unwrap();
let albums = rp
.query()
.music_artist_albums(id, None, Some(AlbumOrder::Recency))
.await
.unwrap();
let albums_expect = artist
.albums
.iter()
@ -1758,7 +1763,7 @@ async fn music_artist_not_found(rp: RustyPipe) {
async fn music_artist_albums_not_found(rp: RustyPipe) {
let err = rp
.query()
.music_artist_albums("UC7cl4MmM6ZZ2TcFyMk_b4pq")
.music_artist_albums("UC7cl4MmM6ZZ2TcFyMk_b4pq", None, None)
.await
.unwrap_err();
@ -2636,23 +2641,6 @@ async fn music_genre_not_found(rp: RustyPipe) {
);
}
//#AB TESTS
const VISITOR_DATA_SEARCH_CHANNEL_HANDLES: &str = "CgszYlc1Yk1WZGRCSSjrwOSbBg%3D%3D";
#[tokio::test]
async fn ab3_search_channel_handles() {
let rp = rp_visitor_data(VISITOR_DATA_SEARCH_CHANNEL_HANDLES);
rp.query()
.search_filter::<YouTubeItem, _>(
"test",
&SearchFilter::new().item_type(search_filter::ItemType::Channel),
)
.await
.unwrap();
}
//#MISCELLANEOUS
#[rstest]
@ -2677,6 +2665,25 @@ async fn invalid_ctoken(#[case] ep: ContinuationEndpoint, rp: RustyPipe) {
}
}
#[rstest]
#[tokio::test]
async fn isrc_search_languages(rp: RustyPipe) {
for lang in LANGUAGES {
let tracks = rp
.query()
.lang(lang)
.music_search_tracks("DEUM71602459")
.await
.unwrap();
let working = tracks.items.items.iter().any(|t| t.id == "g0iRiJ_ck48");
assert_eq!(
working,
!matches!(lang, Language::En | Language::EnGb | Language::EnIn),
"lang: {lang}"
);
}
}
//#TESTUTIL
/// Get the language setting from the environment variable
@ -2706,6 +2713,7 @@ fn unlocalized(lang: Language) -> bool {
lang == Language::En
}
/*
/// Get a new RustyPipe instance with pre-set visitor data
fn rp_visitor_data(vdata: &str) -> RustyPipe {
RustyPipe::builder()
@ -2713,7 +2721,7 @@ fn rp_visitor_data(vdata: &str) -> RustyPipe {
.visitor_data(vdata)
.build()
.unwrap()
}
}*/
/// Assert equality within 10% margin
#[track_caller]