Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: fetch artist albums continuation

Browse source
This commit is contained in:
ThetaDev 2024-10-22 23:56:59 +02:00
parent be18d89ea6
commit b589061a40
No known key found for this signature in database
GPG key ID: E319D3C5148D65B6
17 changed files with 50309 additions and 38906 deletions
Download patch file Download diff file Expand all files Collapse all files

61
src/client/mod.rs
View file

@ -38,6 +38,7 @@ use crate::{
cache::{CacheStorage, FileStorage, DEFAULT_CACHE_FILE},
deobfuscate::DeobfData,
error::{Error, ExtractionError},
model::ArtistId,
param::{Country, Language},
report::{FileReporter, Level, Report, Reporter, RustyPipeInfo, DEFAULT_REPORT_DIR},
serializer::MapResult,
@ -1415,7 +1416,7 @@ impl RustyPipeQuery {
}
};
tracing::debug!("mapped response");
tracing::trace!("mapped response");
Ok(RequestResult { res, status, body })
}
@ -1468,10 +1469,9 @@ impl RustyPipeQuery {
/// - `method`: HTTP method
/// - `endpoint`: YouTube API endpoint (`https://www.youtube.com/youtubei/v1/<XYZ>?key=...`)
/// - `body`: Serializable request body to be sent in json format
/// - `visitor_data`: YouTube visitor data cookie
/// - `deobf`: Deobfuscator (is passed to the mapper to deobfuscate stream URLs).
/// - `ctx_src`: Context source (additional parameters for fetching and mapping, used to build the MapRespCtx)
#[allow(clippy::too_many_arguments)]
async fn execute_request_deobf<
async fn execute_request_ctx<
R: DeserializeOwned + MapResponse<M> + Debug,
M,
B: Serialize + ?Sized,
@ -1482,25 +1482,25 @@ impl RustyPipeQuery {
id: &str,
endpoint: &str,
body: &B,
visitor_data: Option<&str>,
deobf: Option<&DeobfData>,
ctx_src: MapRespCtxSource<'_>,
) -> Result<M, Error> {
tracing::debug!("getting {}({})", operation, id);
let request = self
.request_builder(ctype, endpoint, visitor_data)
.await
.json(body)
.build()?;
let ctx = MapRespCtx {
id,
lang: self.opts.lang,
deobf,
visitor_data: visitor_data.or(self.opts.visitor_data.as_deref()),
deobf: ctx_src.deobf,
visitor_data: ctx_src.visitor_data.or(self.opts.visitor_data.as_deref()),
client_type: ctype,
artist: ctx_src.artist,
};
let request = self
.request_builder(ctype, endpoint, ctx.visitor_data)
.await
.json(body)
.build()?;
let req_res = self.yt_request::<R, M>(&request, &ctx).await?;
// Uncomment to debug response text
@ -1533,7 +1533,7 @@ impl RustyPipeQuery {
operation: &format!("{operation}({id})"),
error,
msgs,
deobf_data: deobf.cloned(),
deobf_data: ctx.deobf.cloned(),
http_request: crate::report::HTTPRequest {
url: request.url().as_str(),
method: request.method().as_str(),
@ -1587,10 +1587,18 @@ impl RustyPipeQuery {
endpoint: &str,
body: &B,
) -> Result<M, Error> {
self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None, None)
.await
self.execute_request_ctx::<R, M, B>(
ctype,
operation,
id,
endpoint,
body,
MapRespCtxSource::default(),
)
.await
}
/*
/// Execute a request to the YouTube API, then map the response.
///
/// Creates a report in case of failure for easy debugging.
@ -1629,6 +1637,7 @@ impl RustyPipeQuery {
)
.await
}
*/
/// Execute a request to the YouTube API and return the response string
///
@ -1664,6 +1673,14 @@ struct MapRespCtx<'a> {
deobf: Option<&'a DeobfData>,
visitor_data: Option<&'a str>,
client_type: ClientType,
artist: Option<ArtistId>,
}
#[derive(Default)]
struct MapRespCtxSource<'a> {
visitor_data: Option<&'a str>,
deobf: Option<&'a DeobfData>,
artist: Option<ArtistId>,
}
impl<'a> MapRespCtx<'a> {
@ -1676,6 +1693,16 @@ impl<'a> MapRespCtx<'a> {
deobf: None,
visitor_data: None,
client_type: ClientType::Desktop,
artist: None,
}
}
}
impl<'a> MapRespCtxSource<'a> {
fn visitor_data(visitor_data: &'a str) -> Self {
Self {
visitor_data: Some(visitor_data),
..Default::default()
}
}
}

164
src/client/music_artist.rs
View file

@ -5,16 +5,19 @@ use regex::Regex;
use tracing::debug;
use crate::{
client::response::url_endpoint::NavigationEndpoint,
client::{response::url_endpoint::NavigationEndpoint, MapRespCtxSource, QContinuation},
error::{Error, ExtractionError},
model::{AlbumItem, ArtistId, MusicArtist},
model::{
paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistId, MusicArtist, MusicItem,
},
param::{AlbumFilter, AlbumOrder},
serializer::MapResult,
util,
util::{self, ProtoBuilder},
};
use super::{
response::{self, music_item::MusicListMapper, url_endpoint::PageType},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
ClientType, MapRespCtx, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
};
impl RustyPipeQuery {
@ -56,7 +59,9 @@ impl RustyPipeQuery {
.await?;
if can_fetch_more {
artist.albums = self.music_artist_albums(artist_id).await?;
artist.albums = self
.music_artist_albums(artist_id, None, Some(AlbumOrder::Recency))
.await?;
}
Ok(artist)
@ -73,21 +78,62 @@ impl RustyPipeQuery {
}
/// Get a list of all albums of a YouTube Music artist
pub async fn music_artist_albums(&self, artist_id: &str) -> Result<Vec<AlbumItem>, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
pub async fn music_artist_albums(
&self,
artist_id: &str,
filter: Option<AlbumFilter>,
order: Option<AlbumOrder>,
) -> Result<Vec<AlbumItem>, Error> {
let visitor_data = self.get_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QBrowseParams {
context: context.clone(),
browse_id: &format!("{}{}", util::ARTIST_DISCOGRAPHY_PREFIX, artist_id),
params: &albums_param(filter, order),
};
self.execute_request::<response::MusicArtistAlbums, _, _>(
ClientType::DesktopMusic,
"music_artist_albums",
artist_id,
"browse",
&request_body,
)
.await
let first_page = self
.execute_request_ctx::<response::MusicArtistAlbums, _, _>(
ClientType::DesktopMusic,
"music_artist_albums",
artist_id,
"browse",
&request_body,
MapRespCtxSource::visitor_data(&visitor_data),
)
.await?;
let mut albums = first_page.albums;
let mut ctoken = first_page.ctoken;
while let Some(tkn) = &ctoken {
let request_body = QContinuation {
context: context.clone(),
continuation: tkn,
};
let resp: Paginator<MusicItem> = self
.execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
ClientType::DesktopMusic,
"music_artist_albums_cont",
artist_id,
"browse",
&request_body,
MapRespCtxSource {
artist: Some(first_page.artist.clone()),
visitor_data: Some(&visitor_data),
..Default::default()
},
)
.await?;
if resp.items.is_empty() {
tracing::warn!("artist albums [{artist_id}] empty continuation");
}
ctoken = resp.ctoken;
albums.extend(resp.items.into_iter().filter_map(AlbumItem::from_ytm_item));
}
Ok(albums)
}
}
@ -280,11 +326,18 @@ fn map_artist_page(
})
}
impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
#[derive(Debug)]
struct FirstAlbumPage {
albums: Vec<AlbumItem>,
ctoken: Option<String>,
artist: ArtistId,
}
impl MapResponse<FirstAlbumPage> for response::MusicArtistAlbums {
fn map_response(
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> {
) -> Result<MapResult<FirstAlbumPage>, ExtractionError> {
// dbg!(&self);
let Some(header) = self.header else {
@ -306,27 +359,55 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
.section_list_renderer
.contents;
let mut mapper = MusicListMapper::with_artist(
ctx.lang,
ArtistId {
id: Some(ctx.id.to_owned()),
name: header.music_header_renderer.title,
},
);
let artist_id = ArtistId {
id: Some(ctx.id.to_owned()),
name: header.music_header_renderer.title,
};
let mut mapper = MusicListMapper::with_artist(ctx.lang, artist_id.clone());
let mut ctoken = None;
for grid in grids {
mapper.map_response(grid.grid_renderer.items);
if ctoken.is_none() {
ctoken = grid
.grid_renderer
.continuations
.into_iter()
.next()
.map(|g| g.next_continuation_data.continuation);
}
}
let mapped = mapper.group_items();
Ok(MapResult {
c: mapped.c.albums,
c: FirstAlbumPage {
albums: mapped.c.albums,
ctoken,
artist: artist_id,
},
warnings: mapped.warnings,
})
}
}
fn albums_param(filter: Option<AlbumFilter>, order: Option<AlbumOrder>) -> String {
let mut pb_filter = ProtoBuilder::new();
if let Some(filter) = filter {
pb_filter.varint(1, filter as u64);
}
if let Some(order) = order {
pb_filter.varint(2, order as u64);
}
pb_filter.bytes(3, &[1, 2]);
let mut pb_48 = ProtoBuilder::new();
pb_48.embedded(15, pb_filter);
let mut pb_3 = ProtoBuilder::new();
pb_3.embedded(48, pb_48);
pb_3.to_base64()
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader};
@ -366,11 +447,12 @@ mod tests {
);
assert_eq!(can_fetch_more, album_page_path.is_some());
// Album overview
if let Some(album_page_path) = album_page_path {
let json_file = File::open(album_page_path).unwrap();
let resp: response::MusicArtistAlbums =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let mut map_res: MapResult<Vec<AlbumItem>> =
let map_res: MapResult<FirstAlbumPage> =
resp.map_response(&MapRespCtx::test(id)).unwrap();
assert!(
@ -378,7 +460,29 @@ mod tests {
"deserialization/mapping warnings: {:?}",
map_res.warnings
);
artist.albums.append(&mut map_res.c);
artist.albums = map_res.c.albums;
// Album overview continuation
for i in 2..10 {
let cont_path =
path!(*TESTFILES / "music_artist" / format!("artist_{name}_{i}.json"));
if !cont_path.is_file() {
break;
}
let json_file = File::open(cont_path).unwrap();
let resp: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> =
resp.map_response(&MapRespCtx::test(id)).unwrap();
assert!(!map_res.c.items.is_empty());
artist.albums.extend(
map_res
.c
.items
.into_iter()
.filter_map(AlbumItem::from_ytm_item),
);
}
}
insta::assert_ron_snapshot!(format!("map_music_artist_{name}"), artist);

6
src/client/music_details.rs
View file

@ -16,7 +16,7 @@ use super::{
self,
music_item::{map_queue_item, MusicListMapper},
},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery, YTContext,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
@ -134,13 +134,13 @@ impl RustyPipeQuery {
tuner_setting_value: "AUTOMIX_SETTING_NORMAL",
};
self.execute_request_vdata::<response::MusicDetails, _, _>(
self.execute_request_ctx::<response::MusicDetails, _, _>(
ClientType::DesktopMusic,
"music_radio",
radio_id,
"next",
&request_body,
Some(&visitor_data),
MapRespCtxSource::visitor_data(&visitor_data),
)
.await
}

9
src/client/music_playlist.rs
View file

@ -17,7 +17,7 @@ use super::{
self,
music_item::{map_album_type, map_artist_id, map_artists, MusicListMapper},
},
ClientType, MapRespCtx, MapResponse, QBrowse, RustyPipeQuery,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QBrowse, RustyPipeQuery,
};
impl RustyPipeQuery {
@ -42,13 +42,16 @@ impl RustyPipeQuery {
browse_id: &format!("VL{playlist_id}"),
};
self.execute_request_vdata::<response::MusicPlaylist, _, _>(
self.execute_request_ctx::<response::MusicPlaylist, _, _>(
ClientType::DesktopMusic,
"music_playlist",
playlist_id,
"browse",
&request_body,
visitor_data.as_deref(),
MapRespCtxSource {
visitor_data: visitor_data.as_deref(),
..Default::default()
},
)
.await
}

53
src/client/pagination.rs
View file

@ -10,7 +10,9 @@ use crate::model::{
use crate::serializer::MapResult;
use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanelVideo};
use super::{response, ClientType, MapRespCtx, MapResponse, QContinuation, RustyPipeQuery};
use super::{
response, ClientType, MapRespCtx, MapRespCtxSource, MapResponse, QContinuation, RustyPipeQuery,
};
impl RustyPipeQuery {
/// Get more YouTube items from the given continuation token and endpoint
@ -37,13 +39,13 @@ impl RustyPipeQuery {
};
let p = self
.execute_request_vdata::<response::MusicContinuation, Paginator<MusicItem>, _>(
.execute_request_ctx::<response::MusicContinuation, Paginator<MusicItem>, _>(
ClientType::DesktopMusic,
"music_continuation",
ctoken,
endpoint.as_str(),
&request_body,
Some(&visitor_data),
MapRespCtxSource::visitor_data(&visitor_data),
)
.await?;
@ -138,7 +140,11 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
self,
ctx: &MapRespCtx<'_>,
) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> {
let mut mapper = MusicListMapper::new(ctx.lang);
let mut mapper = if let Some(artist) = &ctx.artist {
MusicListMapper::with_artist(ctx.lang, artist.clone())
} else {
MusicListMapper::new(ctx.lang)
};
let mut continuations = Vec::new();
match self.continuation_contents {
@ -156,7 +162,11 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
response::music_item::ItemSection::MusicCarouselShelfRenderer(shelf) => {
mapper.map_response(shelf.contents);
}
_ => {}
response::music_item::ItemSection::GridRenderer(mut grid) => {
mapper.map_response(grid.items);
continuations.append(&mut grid.continuations);
}
response::music_item::ItemSection::None => {}
}
}
}
@ -173,6 +183,10 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
}
});
}
Some(response::music_item::ContinuationContents::GridContinuation(mut grid)) => {
mapper.map_response(grid.items);
continuations.append(&mut grid.continuations);
}
None => {}
}
@ -257,6 +271,19 @@ impl<T: FromYtItem> Paginator<T> {
}
Ok(())
}
/// Extend the items of the paginator until the paginator is exhausted.
pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(&mut self, query: Q) -> Result<(), Error> {
let query = query.as_ref();
loop {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
Ok(())
}
}
impl Paginator<Comment> {
@ -334,6 +361,22 @@ macro_rules! paginator {
}
Ok(())
}
/// Extend the items of the paginator until the paginator is exhausted.
pub async fn extend_all<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
) -> Result<(), Error> {
let query = query.as_ref();
loop {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
}
}
Ok(())
}
}
};
}

11
src/client/player.rs
View file

@ -24,7 +24,7 @@ use super::{
self,
player::{self, Format},
},
ClientType, MapRespCtx, MapResponse, MapResult, RustyPipeQuery, YTContext,
ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, RustyPipeQuery, YTContext,
DEFAULT_PLAYER_CLIENT_ORDER,
};
@ -162,14 +162,16 @@ impl RustyPipeQuery {
}
};
self.execute_request_deobf::<response::Player, _, _>(
self.execute_request_ctx::<response::Player, _, _>(
client_type,
"player",
video_id,
"player",
&request_body,
None,
Some(&deobf),
MapRespCtxSource {
deobf: Some(&deobf),
..Default::default()
},
)
.await
}
@ -763,6 +765,7 @@ mod tests {
deobf: Some(&DEOBF_DATA),
visitor_data: None,
client_type,
artist: None,
})
.unwrap();

12
src/client/playlist.rs
View file

@ -13,7 +13,10 @@ use crate::{
util::{self, dictionary, timeago, TryRemove},
};
use super::{response, ClientType, MapRespCtx, MapResponse, MapResult, QBrowse, RustyPipeQuery};
use super::{
response, ClientType, MapRespCtx, MapRespCtxSource, MapResponse, MapResult, QBrowse,
RustyPipeQuery,
};
impl RustyPipeQuery {
/// Get a YouTube playlist
@ -34,13 +37,16 @@ impl RustyPipeQuery {
browse_id: &format!("VL{playlist_id}"),
};
self.execute_request_vdata::<response::Playlist, _, _>(
self.execute_request_ctx::<response::Playlist, _, _>(
ClientType::Desktop,
"playlist",
playlist_id,
"browse",
&request_body,
visitor_data.as_deref(),
MapRespCtxSource {
visitor_data: visitor_data.as_deref(),
..Default::default()
},
)
.await
}

5
src/client/response/music_item.rs
View file

@ -335,6 +335,7 @@ pub(crate) enum ContinuationContents {
MusicShelfContinuation(MusicShelf),
SectionListContinuation(ContentsRenderer<ItemSection>),
PlaylistPanelContinuation(PlaylistPanelRenderer),
GridContinuation(GridRenderer),
}
#[derive(Debug, Deserialize)]
@ -381,11 +382,15 @@ pub(crate) struct Grid {
pub grid_renderer: GridRenderer,
}
#[serde_as]
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct GridRenderer {
pub items: MapResult<Vec<MusicResponseItem>>,
pub header: Option<GridHeader>,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
pub continuations: Vec<MusicContinuationData>,
}
#[derive(Debug, Deserialize)]

3172
src/client/snapshots/rustypipe__client__music_artist__tests__map_music_artist_default.snap
View file

File diff suppressed because it is too large Load diff

834
src/client/snapshots/rustypipe__client__music_artist__tests__map_music_artist_no_cont.snap
View file

File diff suppressed because it is too large Load diff

23
src/param/mod.rs
View file

@ -23,9 +23,10 @@ pub enum ChannelVideoTab {
}
/// Sort order for channel videos
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChannelOrder {
/// Order videos with the latest upload date first (default)
#[default]
Latest = 1,
/// Order videos with the highest number of views first
Popular = 2,
@ -43,3 +44,23 @@ impl ChannelVideoTab {
}
}
}
/// Sort order for YTM artist albums
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AlbumOrder {
/// Sort albums by release date
Recency = 1,
/// Sort albums by popularity
Popularity = 2,
/// Sort albums by their name
Alphabetical = 3,
}
/// Filter for YTM artist albums
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AlbumFilter {
/// Only show albums
Albums = 1,
/// Only show singles
Singles = 2,
}

2
src/param/search_filter.rs
View file

@ -204,7 +204,7 @@ impl SearchFilter {
if let Some(sort) = self.sort {
pb.varint(1, sort as u64);
}
if !filters.bytes.is_empty() {
if !filters.is_empty() {
pb.embedded(2, filters);
}
if self.verbatim {

14
src/util/protobuf.rs
View file

@ -1,7 +1,7 @@
/// [`ProtoBuilder`] is used to construct protobuf messages using a builder pattern
#[derive(Debug, Default)]
pub struct ProtoBuilder {
pub bytes: Vec<u8>,
bytes: Vec<u8>,
}
impl ProtoBuilder {
@ -39,6 +39,11 @@ impl ProtoBuilder {
self._varint(val);
}
/// Returns `true` if the builder contains no data
pub fn is_empty(&self) -> bool {
self.bytes.is_empty()
}
/// Write a varint field
pub fn varint(&mut self, field: u32, val: u64) {
self._field(field, 0);
@ -52,6 +57,13 @@ impl ProtoBuilder {
self.bytes.extend_from_slice(string.as_bytes());
}
/// Write a bytes field
pub fn bytes(&mut self, field: u32, bytes: &[u8]) {
self._field(field, 2);
self._varint(bytes.len() as u64);
self.bytes.extend_from_slice(bytes);
}
/// Write an embedded message
///
/// Requires passing another [`ProtoBuilder`] with the embedded message.