From f94d8db4d0553ea7a5a08d5a7be63c591b92e616 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 27 Jan 2023 19:49:16 +0100 Subject: [PATCH] feat: add logging for all operations fix: music_artist: fetch visitor data only once --- src/client/mod.rs | 24 ++++--- src/client/music_artist.rs | 124 ++++++++++++++++++------------------- src/deobfuscate.rs | 3 +- tests/youtube.rs | 7 ++- 4 files changed, 78 insertions(+), 80 deletions(-) diff --git a/src/client/mod.rs b/src/client/mod.rs index b454b8e..7dd8a4f 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -25,7 +25,6 @@ mod channel_rss; use std::sync::Arc; use std::{borrow::Cow, fmt::Debug}; -use log::{debug, error, warn}; use once_cell::sync::Lazy; use rand::Rng; use regex::Regex; @@ -323,6 +322,7 @@ impl RustyPipeBuilder { .user_agent(self.user_agent) .gzip(true) .brotli(true) + .redirect(reqwest::redirect::Policy::none()) .build() .unwrap(); @@ -331,7 +331,7 @@ impl RustyPipeBuilder { match serde_json::from_str::(&data) { Ok(data) => data, Err(e) => { - error!("Could not deserialize cache. Error: {}", e); + log::error!("Could not deserialize cache. Error: {}", e); CacheData::default() } } @@ -523,7 +523,7 @@ impl RustyPipe { }; let ms = util::retry_delay(n, 1000, 60000, 3); - warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); + log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await; last_res = Some(res); @@ -656,7 +656,7 @@ impl RustyPipe { match desktop_client.get() { Some(cdata) => cdata.version.to_owned(), None => { - debug!("getting desktop client version"); + log::debug!("getting desktop client version"); match self.extract_desktop_client_version().await { Ok(version) => { *desktop_client = CacheEntry::from(ClientData { @@ -667,7 +667,7 @@ impl RustyPipe { version } Err(e) => { - warn!("{}, falling back to hardcoded version", e); + log::warn!("{}, falling back to hardcoded version", e); DESKTOP_CLIENT_VERSION.to_owned() } } @@ -688,7 +688,7 @@ impl RustyPipe { match music_client.get() { Some(cdata) => cdata.version.to_owned(), None => { - debug!("getting music client version"); + log::debug!("getting music client version"); match self.extract_music_client_version().await { Ok(version) => { *music_client = CacheEntry::from(ClientData { @@ -699,7 +699,7 @@ impl RustyPipe { version } Err(e) => { - warn!("{}, falling back to hardcoded version", e); + log::warn!("{}, falling back to hardcoded version", e); DESKTOP_MUSIC_CLIENT_VERSION.to_owned() } } @@ -715,7 +715,7 @@ impl RustyPipe { match deobf.get() { Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), None => { - debug!("getting deobfuscator"); + log::debug!("getting deobfuscator"); let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?; *deobf = CacheEntry::from(new_deobf.get_data()); drop(deobf); @@ -736,12 +736,13 @@ impl RustyPipe { match serde_json::to_string(&cdata) { Ok(data) => storage.write(&data), - Err(e) => error!("Could not serialize cache. Error: {}", e), + Err(e) => log::error!("Could not serialize cache. Error: {}", e), } } } async fn get_ytm_visitor_data(&self) -> Result { + log::debug!("getting YTM visitor data"); let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?; resp.headers() @@ -1040,6 +1041,8 @@ impl RustyPipeQuery { body: &B, deobf: Option<&Deobfuscator>, ) -> Result { + log::debug!("getting {}({})", operation, id); + let request = self .request_builder(ctype, endpoint) .await @@ -1212,7 +1215,7 @@ trait MapResponse { fn validate_country(country: Country) -> Country { if country == Country::Zz { - warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); + log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); Country::Us } else { country @@ -1227,6 +1230,7 @@ mod tests { async fn t_get_ytm_visitor_data() { let rp = RustyPipe::new(); let visitor_data = rp.get_ytm_visitor_data().await.unwrap(); + dbg!(&visitor_data); assert!(visitor_data.ends_with("%3D")); assert_eq!(visitor_data.len(), 32) } diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 1d21d42..3319316 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -35,88 +35,82 @@ impl RustyPipeQuery { all_albums: bool, ) -> Result { let artist_id = artist_id.as_ref(); - log::debug!("Getting music artist {}", artist_id); + let visitor_data = match all_albums { + true => Some(self.get_ytm_visitor_data().await?), + false => None, + }; - let res = self._music_artist(artist_id, all_albums).await; + let res = self._music_artist(artist_id, visitor_data.as_deref()).await; if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res { - log::debug!("Music artist {} redirects to {}", artist_id, &id); - self._music_artist(&id, all_albums).await.map(|x| *x) + log::debug!("music artist {} redirects to {}", artist_id, &id); + self._music_artist(&id, visitor_data.as_deref()).await } else { - res.map(|x| *x) + res } } - async fn _music_artist>( + async fn _music_artist( &self, - artist_id: S, - all_albums: bool, - ) -> Result, Error> { - let artist_id = artist_id.as_ref(); + artist_id: &str, + all_albums_vdata: Option<&str>, + ) -> Result { + match all_albums_vdata { + Some(visitor_data) => { + let context = self + .get_context(ClientType::DesktopMusic, true, Some(visitor_data)) + .await; + let request_body = QBrowse { + context, + browse_id: artist_id, + }; - if all_albums { - let visitor_data = self.get_ytm_visitor_data().await?; - let context = self - .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) - .await; - let request_body = QBrowse { - context, - browse_id: artist_id, - }; + let (mut artist, album_page_params) = self + .execute_request::( + ClientType::DesktopMusic, + "music_artist", + artist_id, + "browse", + &request_body, + ) + .await?; - let (mut artist, album_page_params) = self - .execute_request::( + let visitor_data = Rc::new(visitor_data); + let album_page_results = stream::iter(album_page_params) + .map(|params| { + let visitor_data = visitor_data.clone(); + async move { + self.music_artist_album_page(artist_id, ¶ms, &visitor_data) + .await + } + }) + .buffer_unordered(2) + .collect::>() + .await; + + for res in album_page_results { + let mut res = res?; + artist.albums.append(&mut res); + } + + Ok(artist) + } + None => { + let context = self.get_context(ClientType::DesktopMusic, true, None).await; + let request_body = QBrowse { + context, + browse_id: artist_id, + }; + + self.execute_request::( ClientType::DesktopMusic, "music_artist", artist_id, "browse", &request_body, ) - .await?; - - let visitor_data = Rc::new(visitor_data); - let n_album_pages = album_page_params.len(); - let album_page_results = stream::iter(album_page_params) - .enumerate() - .map(|(i, params)| { - let visitor_data = visitor_data.clone(); - log::debug!( - "Getting music artist {} section {}/{}", - artist_id, - i + 1, - n_album_pages - ); - async move { - self.music_artist_album_page(artist_id, ¶ms, &visitor_data) - .await - } - }) - .buffer_unordered(2) - .collect::>() - .await; - - for res in album_page_results { - let mut res = res?; - artist.albums.append(&mut res); + .await } - - Ok(artist.into()) - } else { - let context = self.get_context(ClientType::DesktopMusic, true, None).await; - let request_body = QBrowse { - context, - browse_id: artist_id, - }; - - self.execute_request::( - ClientType::DesktopMusic, - "music_artist", - artist_id, - "browse", - &request_body, - ) - .await - .map(|x: MusicArtist| x.into()) } } diff --git a/src/deobfuscate.rs b/src/deobfuscate.rs index 9753e95..995da6a 100644 --- a/src/deobfuscate.rs +++ b/src/deobfuscate.rs @@ -1,5 +1,4 @@ use fancy_regex::Regex as FancyRegex; -use log::debug; use once_cell::sync::Lazy; use regex::Regex; use reqwest::Client; @@ -27,7 +26,7 @@ impl Deobfuscator { let js_url = get_player_js_url(&http).await?; let player_js = get_response(&http, &js_url).await?; - debug!("Downloaded player.js from {}", js_url); + log::debug!("downloaded player.js from {}", js_url); let sig_fn = get_sig_fn(&player_js)?; let nsig_fn = get_nsig_fn(&player_js)?; diff --git a/tests/youtube.rs b/tests/youtube.rs index 398a3de..0e70625 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1424,8 +1424,8 @@ async fn music_album_not_found() { #[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg", false, 0, 2)] // querying Trailerpark's secondary YouTube channel should result in the YTM channel being fetched #[case::secondary_channel("no_more_albums", "UCC9192yGQD25eBZgFZ84MPw", true, 15, 0)] -#[tokio::test] -async fn music_artist( +#[test_log::test] +fn music_artist( #[case] name: &str, #[case] id: &str, #[case] all_albums: bool, @@ -1433,7 +1433,8 @@ async fn music_artist( #[case] min_playlists: usize, ) { let rp = RustyPipe::builder().strict().build(); - let mut artist = rp.query().music_artist(id, all_albums).await.unwrap(); + + let mut artist = tokio_test::block_on(rp.query().music_artist(id, all_albums)).unwrap(); assert_gte(artist.tracks.len(), min_tracks, "tracks"); assert_gte(artist.playlists.len(), min_playlists, "playlists");