Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add logging for all operations

Browse source 
fix: music_artist: fetch visitor data only once
This commit is contained in:
ThetaDev 2023-01-27 19:49:16 +01:00
parent be741d28ca
commit f94d8db4d0
4 changed files with 78 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

24
src/client/mod.rs
View file

@ -25,7 +25,6 @@ mod channel_rss;
use std::sync::Arc;
use std::{borrow::Cow, fmt::Debug};
use log::{debug, error, warn};
use once_cell::sync::Lazy;
use rand::Rng;
use regex::Regex;
@ -323,6 +322,7 @@ impl RustyPipeBuilder {
.user_agent(self.user_agent)
.gzip(true)
.brotli(true)
.redirect(reqwest::redirect::Policy::none())
.build()
.unwrap();
@ -331,7 +331,7 @@ impl RustyPipeBuilder {
match serde_json::from_str::<CacheData>(&data) {
Ok(data) => data,
Err(e) => {
error!("Could not deserialize cache. Error: {}", e);
log::error!("Could not deserialize cache. Error: {}", e);
CacheData::default()
}
}
@ -523,7 +523,7 @@ impl RustyPipe {
};
let ms = util::retry_delay(n, 1000, 60000, 3);
warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
log::warn!("Retry attempt #{}. Error: {}. Waiting {} ms", n, emsg, ms);
tokio::time::sleep(std::time::Duration::from_millis(ms.into())).await;
last_res = Some(res);
@ -656,7 +656,7 @@ impl RustyPipe {
match desktop_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => {
debug!("getting desktop client version");
log::debug!("getting desktop client version");
match self.extract_desktop_client_version().await {
Ok(version) => {
*desktop_client = CacheEntry::from(ClientData {
@ -667,7 +667,7 @@ impl RustyPipe {
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_CLIENT_VERSION.to_owned()
}
}
@ -688,7 +688,7 @@ impl RustyPipe {
match music_client.get() {
Some(cdata) => cdata.version.to_owned(),
None => {
debug!("getting music client version");
log::debug!("getting music client version");
match self.extract_music_client_version().await {
Ok(version) => {
*music_client = CacheEntry::from(ClientData {
@ -699,7 +699,7 @@ impl RustyPipe {
version
}
Err(e) => {
warn!("{}, falling back to hardcoded version", e);
log::warn!("{}, falling back to hardcoded version", e);
DESKTOP_MUSIC_CLIENT_VERSION.to_owned()
}
}
@ -715,7 +715,7 @@ impl RustyPipe {
match deobf.get() {
Some(deobf) => Ok(Deobfuscator::from(deobf.to_owned())),
None => {
debug!("getting deobfuscator");
log::debug!("getting deobfuscator");
let new_deobf = Deobfuscator::new(self.inner.http.clone()).await?;
*deobf = CacheEntry::from(new_deobf.get_data());
drop(deobf);
@ -736,12 +736,13 @@ impl RustyPipe {
match serde_json::to_string(&cdata) {
Ok(data) => storage.write(&data),
Err(e) => error!("Could not serialize cache. Error: {}", e),
Err(e) => log::error!("Could not serialize cache. Error: {}", e),
}
}
}
async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
log::debug!("getting YTM visitor data");
let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
resp.headers()
@ -1040,6 +1041,8 @@ impl RustyPipeQuery {
body: &B,
deobf: Option<&Deobfuscator>,
) -> Result<M, Error> {
log::debug!("getting {}({})", operation, id);
let request = self
.request_builder(ctype, endpoint)
.await
@ -1212,7 +1215,7 @@ trait MapResponse<T> {
fn validate_country(country: Country) -> Country {
if country == Country::Zz {
warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
log::warn!("Country:Zz (Global) can only be used for fetching music charts, falling back to Country:Us");
Country::Us
} else {
country
@ -1227,6 +1230,7 @@ mod tests {
async fn t_get_ytm_visitor_data() {
let rp = RustyPipe::new();
let visitor_data = rp.get_ytm_visitor_data().await.unwrap();
dbg!(&visitor_data);
assert!(visitor_data.ends_with("%3D"));
assert_eq!(visitor_data.len(), 32)
}

124
src/client/music_artist.rs
View file

@ -35,88 +35,82 @@ impl RustyPipeQuery {
all_albums: bool,
) -> Result<MusicArtist, Error> {
let artist_id = artist_id.as_ref();
log::debug!("Getting music artist {}", artist_id);
let visitor_data = match all_albums {
true => Some(self.get_ytm_visitor_data().await?),
false => None,
};
let res = self._music_artist(artist_id, all_albums).await;
let res = self._music_artist(artist_id, visitor_data.as_deref()).await;
if let Err(Error::Extraction(ExtractionError::Redirect(id))) = res {
log::debug!("Music artist {} redirects to {}", artist_id, &id);
self._music_artist(&id, all_albums).await.map(|x| *x)
log::debug!("music artist {} redirects to {}", artist_id, &id);
self._music_artist(&id, visitor_data.as_deref()).await
} else {
res.map(|x| *x)
res
}
}
async fn _music_artist<S: AsRef<str>>(
async fn _music_artist(
&self,
artist_id: S,
all_albums: bool,
) -> Result<Box<MusicArtist>, Error> {
let artist_id = artist_id.as_ref();
artist_id: &str,
all_albums_vdata: Option<&str>,
) -> Result<MusicArtist, Error> {
match all_albums_vdata {
Some(visitor_data) => {
let context = self
.get_context(ClientType::DesktopMusic, true, Some(visitor_data))
.await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
if all_albums {
let visitor_data = self.get_ytm_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
let (mut artist, album_page_params) = self
.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await?;
let (mut artist, album_page_params) = self
.execute_request::<response::MusicArtist, _, _>(
let visitor_data = Rc::new(visitor_data);
let album_page_results = stream::iter(album_page_params)
.map(|params| {
let visitor_data = visitor_data.clone();
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
}
Ok(artist)
}
None => {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await?;
let visitor_data = Rc::new(visitor_data);
let n_album_pages = album_page_params.len();
let album_page_results = stream::iter(album_page_params)
.enumerate()
.map(|(i, params)| {
let visitor_data = visitor_data.clone();
log::debug!(
"Getting music artist {} section {}/{}",
artist_id,
i + 1,
n_album_pages
);
async move {
self.music_artist_album_page(artist_id, &params, &visitor_data)
.await
}
})
.buffer_unordered(2)
.collect::<Vec<_>>()
.await;
for res in album_page_results {
let mut res = res?;
artist.albums.append(&mut res);
.await
}
Ok(artist.into())
} else {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
context,
browse_id: artist_id,
};
self.execute_request::<response::MusicArtist, _, _>(
ClientType::DesktopMusic,
"music_artist",
artist_id,
"browse",
&request_body,
)
.await
.map(|x: MusicArtist| x.into())
}
}

3
src/deobfuscate.rs
View file

@ -1,5 +1,4 @@
use fancy_regex::Regex as FancyRegex;
use log::debug;
use once_cell::sync::Lazy;
use regex::Regex;
use reqwest::Client;
@ -27,7 +26,7 @@ impl Deobfuscator {
let js_url = get_player_js_url(&http).await?;
let player_js = get_response(&http, &js_url).await?;
debug!("Downloaded player.js from {}", js_url);
log::debug!("downloaded player.js from {}", js_url);
let sig_fn = get_sig_fn(&player_js)?;
let nsig_fn = get_nsig_fn(&player_js)?;

7
tests/youtube.rs
View file

@ -1424,8 +1424,8 @@ async fn music_album_not_found() {
#[case::no_artist("no_artist", "UCh8gHdtzO2tXd593_bjErWg", false, 0, 2)]
// querying Trailerpark's secondary YouTube channel should result in the YTM channel being fetched
#[case::secondary_channel("no_more_albums", "UCC9192yGQD25eBZgFZ84MPw", true, 15, 0)]
#[tokio::test]
async fn music_artist(
#[test_log::test]
fn music_artist(
#[case] name: &str,
#[case] id: &str,
#[case] all_albums: bool,
@ -1433,7 +1433,8 @@ async fn music_artist(
#[case] min_playlists: usize,
) {
let rp = RustyPipe::builder().strict().build();
let mut artist = rp.query().music_artist(id, all_albums).await.unwrap();
let mut artist = tokio_test::block_on(rp.query().music_artist(id, all_albums)).unwrap();
assert_gte(artist.tracks.len(), min_tracks, "tracks");
assert_gte(artist.playlists.len(), min_playlists, "playlists");