Sulkta-OSS/rustypipe
Archived
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add logging for all operations

Browse source 
fix: music_artist: fetch visitor data only once
This commit is contained in:
ThetaDev 2023-01-27 19:49:16 +01:00
parent be741d28ca
commit f94d8db4d0
4 changed files with 78 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

24
src/client/mod.rs
View file

@ -25,7 +25,6 @@ mod channel_rss;
use std::sync::Arc; use std::sync::Arc;
use std::{borrow::Cow, fmt::Debug}; use std::{borrow::Cow, fmt::Debug};
use log::{debug, error, warn};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use rand::Rng; use rand::Rng;
use regex::Regex; use regex::Regex;
@ -323,6 +322,7 @@ impl RustyPipeBuilder {
.user_agent(self.user_agent) .user_agent(self.user_agent)
.gzip(true) .gzip(true)
.brotli(true) .brotli(true)
.redirect(reqwest::redirect::Policy::none())
.build() .build()
.unwrap(); .unwrap();
@ -331,7 +331,7 @@ impl RustyPipeBuilder {
match serde_json::from_str::<CacheData>(&data) { match serde_json::from_str::<CacheData>(&data) {
Ok(data) => data, Ok(data) => data,
Err(e) => { Err(e) => {
error!("Could not deserialize cache. Error: {}", e); log::error!("Could not deserialize cache. Error: {}", e);
CacheData::default() CacheData::default()
} }
} }
@ -523,7 +523,7 @@ impl RustyPipe {
}; };
let ms = util::retry_delay(n, 1000, 60000, 3); let ms = util::retry_delay(n, 1000, 60000, 3);
warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms); log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await; tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
last_res = Some(res); last_res = Some(res);
@ -656,7 +656,7 @@ impl RustyPipe {
match desktop_client.get() { match desktop_client.get() {
Some(cdata) => cdata.version.to_owned(), Some(cdata) => cdata.version.to_owned(),
None => { None => {
debug!("getting desktop client version"); log::debug!("getting desktop client version");
match self.extract_desktop_client_version().await { match self.extract_desktop_client_version().await {
Ok(version) => { Ok(version) => {
*desktop_client = CacheEntry::from(ClientData { *desktop_client = CacheEntry::from(ClientData {
@ -667,7 +667,7 @@ impl RustyPipe {
version version
} }
Err(e) => { Err(e) => {
warn!("{}, falling back to hardcoded version", e); log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_CLIENT_VERSION.to_owned() DESKTOP_CLIENT_VERSION.to_owned()
} }
} }
@ -688,7 +688,7 @@ impl RustyPipe {
match music_client.get() { match music_client.get() {
Some(cdata) => cdata.version.to_owned(), Some(cdata) => cdata.version.to_owned(),
None => { None => {
debug!("getting music client version"); log::debug!("getting music client version");
match self.extract_music_client_version().await { match self.extract_music_client_version().await {
Ok(version) => { Ok(version) => {
*music_client = CacheEntry::from(ClientData { *music_client = CacheEntry::from(ClientData {
@ -699,7 +699,7 @@ impl RustyPipe {
version version
} }
Err(e) => { Err(e) => {
warn!("{}, falling back to hardcoded version", e); log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_MUSIC_CLIENT_VERSION.to_owned() DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
} }
} }
@ -715,7 +715,7 @@ impl RustyPipe {
match deobf.get() { match deobf.get() {
Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())), Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())),
None => { None => {
debug!("getting deobfuscator"); log::debug!("getting deobfuscator");
let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?; let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?;
*deobf = CacheEntry::from(new_deobf.get_data()); *deobf = CacheEntry::from(new_deobf.get_data());
drop(deobf); drop(deobf);
@ -736,12 +736,13 @@ impl RustyPipe {
match serde_json::to_string(&cdata) { match serde_json::to_string(&cdata) {
Ok(data) => storage.write(&data), Ok(data) => storage.write(&data),
Err(e) => error!("Could not serialize cache. Error: {}", e), Err(e) => log::error!("Could not serialize cache. Error: {}", e),
} }
} }
} }
async fn get_ytm_visitor_data(&self) -> Result<String, Error> { async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
log::debug!("getting YTM visitor data");
let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?; let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
resp.headers() resp.headers()
@ -1040,6 +1041,8 @@ impl RustyPipeQuery {
body: &B, body: &B,
deobf: Option<&Deobfuscator>, deobf: Option<&Deobfuscator>,
) -> Result<M, Error> { ) -> Result<M, Error> {
log::debug!("getting {}({})", operation, id);
let request = self let request = self
.request_builder(ctype, endpoint) .request_builder(ctype, endpoint)
.await .await
@ -1212,7 +1215,7 @@ trait MapResponse<T> {
fn validate_country(country: Country) -> Country { fn validate_country(country: Country) -> Country {
if country == Country::Zz { if country == Country::Zz {
warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us"); log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
Country::Us Country::Us
} else { } else {
country country
@ -1227,6 +1230,7 @@ mod tests {
async fn t_get_ytm_visitor_data() { async fn t_get_ytm_visitor_data() {
let rp = RustyPipe::new(); let rp = RustyPipe::new();
let visitor_data = rp.get_ytm_visitor_data().await.unwrap(); let visitor_data = rp.get_ytm_visitor_data().await.unwrap();
dbg!(&visitor_data);
assert!(visitor_data.ends_with("%3D")); assert!(visitor_data.ends_with("%3D"));
assert_eq!(visitor_data.len(), 32) assert_eq!(visitor_data.len(), 32)
} }

124
src/client/music_artist.rs
View file

@ -35,88 +35,82 @@ impl RustyPipeQuery {
all_albums: bool, all_albums: bool,
) -> Result<MusicArtist, Error> { ) -> Result<MusicArtist, Error> {
let artist_id = artist_id.as_ref(); let artist_id = artist_id.as_ref();
log::debug!("Getting music artist {}", artist_id); let visitor_data = match all_albums {
true => Some(self.get_ytm_visitor_data().await?),
false => None,
};
let res = self._music_artist(artist_id, all_albums).await; let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res { if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
log::debug!("Music artist {} redirects to {}", artist_id, &id); log::debug!("music artist {} redirects to {}", artist_id, &id);
self._music_artist(&id, all_albums).await.map(|x| *x) self._music_artist(&id, visitor_data.as_deref()).await
} else { } else {
res.map(|x| *x) res
} }
} }
async fn _music_artist<S: AsRef<str>>( async fn _music_artist(
&self, &self,
artist_id: S, artist_id: &str,
all_albums: bool, all_albums_vdata: Option<&str>,
) -> Result<Box<MusicArtist>, Error> { ) -> Result<MusicArtist, Error> {
let artist_id = artist_id.as_ref(); match all_albums_vdata {
Some(visitor_data) => {
let context = self
.get_context(ClientType::DesktopMusic, true, Some(visitor_data))
.await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
if all_albums { let (mut artist, album_page_params) = self
let visitor_data = self.get_ytm_visitor_data().await?; .execute_request::<response::MusicArtist, _, _>(
let context = self ClientType::DesktopMusic,
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) "music_artist",
.await; artist_id,
let request_body = QBrowse { "browse",
context, &request_body,
browse_id: artist_id, )
}; .await?;
let (mut artist, album_page_params) = self let visitor_data = Rc::new(visitor_data);
.execute_request::<response::MusicArtist, _, _>( let album_page_results = stream::iter(album_page_params)
.map(|params| {
let visitor_data = visitor_data.clone();
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
}
Ok(artist)
}
None => {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic, ClientType::DesktopMusic,
"music_artist", "music_artist",
artist_id, artist_id,
"browse", "browse",
&request_body, &request_body,
) )
.await?; .await
let visitor_data = Rc::new(visitor_data);
let n_album_pages = album_page_params.len();
let album_page_results = stream::iter(album_page_params)
.enumerate()
.map(|(i, params)| {
let visitor_data = visitor_data.clone();
log::debug!(
"Getting music artist {} section {}/{}",
artist_id,
i + 1,
n_album_pages
);
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
} }
Ok(artist.into())
} else {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await
.map(|x: MusicArtist| x.into())
} }
} }

3
src/deobfuscate.rs
View file

@ -1,5 +1,4 @@
use fancy_regex::Regex as FancyRegex; use fancy_regex::Regex as FancyRegex;
use log::debug;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use reqwest::Client; use reqwest::Client;
@ -27,7 +26,7 @@ impl Deobfuscator {
let js_url = get_player_js_url(&http).await?; let js_url = get_player_js_url(&http).await?;
let player_js = get_response(&http, &js_url).await?; let player_js = get_response(&http, &js_url).await?;
debug!("Downloaded player.js from {}", js_url); log::debug!("downloaded player.js from {}", js_url);
let sig_fn = get_sig_fn(&player_js)?; let sig_fn = get_sig_fn(&player_js)?;
let nsig_fn = get_nsig_fn(&player_js)?; let nsig_fn = get_nsig_fn(&player_js)?;

7
tests/youtube.rs
View file

@ -1424,8 +1424,8 @@ async fn music_album_not_found() {
#[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg", false, 0, 2)] #[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg", false, 0, 2)]
// querying Trailerpark's secondary YouTube channel should result in the YTM channel being fetched // querying Trailerpark's secondary YouTube channel should result in the YTM channel being fetched
#[case::secondary_channel("no_more_albums", "UCC9192yGQD25eBZgFZ84MPw", true, 15, 0)] #[case::secondary_channel("no_more_albums", "UCC9192yGQD25eBZgFZ84MPw", true, 15, 0)]
#[tokio::test] #[test_log::test]
async fn music_artist( fn music_artist(
#[case] name: &str, #[case] name: &str,
#[case] id: &str, #[case] id: &str,
#[case] all_albums: bool, #[case] all_albums: bool,
@ -1433,7 +1433,8 @@ async fn music_artist(
#[case] min_playlists: usize, #[case] min_playlists: usize,
) { ) {
let rp = RustyPipe::builder().strict().build(); let rp = RustyPipe::builder().strict().build();
let mut artist = rp.query().music_artist(id, all_albums).await.unwrap();
let mut artist = tokio_test::block_on(rp.query().music_artist(id, all_albums)).unwrap();
assert_gte(artist.tracks.len(), min_tracks, "tracks"); assert_gte(artist.tracks.len(), min_tracks, "tracks");
assert_gte(artist.playlists.len(), min_playlists, "playlists"); assert_gte(artist.playlists.len(), min_playlists, "playlists");