Sulkta-OSS/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: increase default timeout to 20s, use ffmpeg copy arg

Browse source 
refactor: use enum for navigation endpoint
This commit is contained in:
ThetaDev 2023-05-17 14:35:10 +02:00
parent dc7bd7befc
commit 805cc5088f
11 changed files with 226 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/client/mod.rs
View file

@ -454,7 +454,7 @@ impl RustyPipeBuilder {
.brotli(true)
.redirect(reqwest::redirect::Policy::none());
if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(10)) {
if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(20)) {
client_builder = client_builder.timeout(timeout);
}
@ -556,7 +556,7 @@ impl RustyPipeBuilder {
/// The timeout is applied from when the request starts connecting until the
/// response body has finished.
///
/// **Default value**: 10s
/// **Default value**: 20s
#[must_use]
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = DefaultOpt::Some(timeout);

56
src/client/music_artist.rs
View file

@ -4,6 +4,7 @@ use once_cell::sync::Lazy;
use regex::Regex;
use crate::{
client::response::url_endpoint::{MusicPageType, NavigationEndpoint},
error::{Error, ExtractionError},
model::{AlbumItem, ArtistId, MusicArtist},
serializer::MapResult,
@ -188,38 +189,29 @@ fn map_artist_page(
.music_carousel_shelf_basic_header_renderer
.more_content_button
{
if let Some(bep) =
button.button_renderer.navigation_endpoint.browse_endpoint
{
if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
match cfg.browse_endpoint_context_music_config.page_type {
// Music videos
PageType::Playlist => {
if videos_playlist_id.is_none() {
videos_playlist_id = Some(bep.browse_id);
}
}
// Albums
PageType::ArtistDiscography => {
match button.button_renderer.navigation_endpoint.music_page() {
// Music videos
Some((MusicPageType::Playlist, id)) => {
if videos_playlist_id.is_none() {
videos_playlist_id = Some(id);
}
}
// Albums
Some((MusicPageType::ArtistDiscography, _)) => {
can_fetch_more = true;
extendable_albums = true;
}
// Albums or playlists
Some((MusicPageType::Artist, _)) => {
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.page_type() {
can_fetch_more = true;
extendable_albums = true;
}
// Albums or playlists
PageType::Artist => {
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
config.browse_endpoint_context_music_config.page_type
})}) {
can_fetch_more = true;
extendable_albums = true;
}
}
}
_ => {}
}
}
_ => {}
}
}
}
@ -251,10 +243,12 @@ fn map_artist_page(
});
let radio_id = header.start_radio_button.and_then(|b| {
b.button_renderer
.navigation_endpoint
.watch_endpoint
.and_then(|w| w.playlist_id)
if let NavigationEndpoint::Watch { watch_endpoint } = b.button_renderer.navigation_endpoint
{
watch_endpoint.playlist_id
} else {
None
}
});
Ok(MapResult {

28
src/client/music_genres.rs
View file

@ -7,7 +7,7 @@ use crate::{
};
use super::{
response::{self, music_item::MusicListMapper},
response::{self, music_item::MusicListMapper, url_endpoint::NavigationEndpoint},
ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
};
@ -144,18 +144,20 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
h.music_carousel_shelf_basic_header_renderer
.more_content_button
.and_then(|btn| {
btn.button_renderer
.navigation_endpoint
.browse_endpoint
.and_then(|browse| {
if browse.browse_id
== "FEmusic_moods_and_genres_category"
{
Some(browse.params)
} else {
None
}
})
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = btn.button_renderer.navigation_endpoint
{
if browse_endpoint.browse_id
== "FEmusic_moods_and_genres_category"
{
Some(browse_endpoint.params)
} else {
None
}
} else {
None
}
})
}),
shelf.contents,

24
src/client/response/music_item.rs
View file

@ -759,7 +759,7 @@ impl MusicListMapper {
}));
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => {
MusicPageType::None | MusicPageType::ArtistDiscography => {
// There may be broken YT channels from the artist search. They can be skipped.
Ok(None)
}
@ -901,7 +901,7 @@ impl MusicListMapper {
}));
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => Ok(None),
MusicPageType::None | MusicPageType::ArtistDiscography => Ok(None),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
@ -1039,7 +1039,7 @@ impl MusicListMapper {
}));
Some(MusicItemType::Playlist)
}
MusicPageType::None => None,
MusicPageType::None | MusicPageType::ArtistDiscography => None,
MusicPageType::Unknown => {
self.has_unknown = true;
None
@ -1171,20 +1171,22 @@ fn map_artist_id_fallback(
pub(crate) fn map_artist_id(entries: Vec<MusicItemMenuEntry>) -> Option<String> {
entries.into_iter().find_map(|i| {
let ep = i
.menu_navigation_item_renderer
.navigation_endpoint
.browse_endpoint;
ep.and_then(|ep| {
ep.browse_endpoint_context_supported_configs
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = i.menu_navigation_item_renderer.navigation_endpoint
{
browse_endpoint
.browse_endpoint_context_supported_configs
.and_then(|cfg| {
if cfg.browse_endpoint_context_music_config.page_type == PageType::Artist {
Some(ep.browse_id)
Some(browse_endpoint.browse_id)
} else {
None
}
})
})
} else {
None
}
})
}

123
src/client/response/url_endpoint.rs
View file

@ -11,21 +11,20 @@ pub(crate) struct ResolvedUrl {
}
#[serde_as]
#[derive(Debug, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub(crate) struct NavigationEndpoint {
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub watch_endpoint: Option<WatchEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub browse_endpoint: Option<BrowseEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub url_endpoint: Option<UrlEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub command_metadata: Option<CommandMetadata>,
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub(crate) enum NavigationEndpoint {
#[serde(rename_all = "camelCase")]
Watch { watch_endpoint: WatchEndpoint },
#[serde(rename_all = "camelCase")]
Browse {
browse_endpoint: BrowseEndpoint,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
command_metadata: Option<CommandMetadata>,
},
#[serde(rename_all = "camelCase")]
Url { url_endpoint: UrlEndpoint },
}
#[derive(Debug, Deserialize)]
@ -184,6 +183,7 @@ impl PageType {
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub(crate) enum MusicPageType {
Artist,
ArtistDiscography,
Album,
Playlist,
Track { is_video: bool },
@ -195,47 +195,82 @@ impl From<PageType> for MusicPageType {
fn from(t: PageType) -> Self {
match t {
PageType::Artist => MusicPageType::Artist,
PageType::ArtistDiscography => MusicPageType::ArtistDiscography,
PageType::Album => MusicPageType::Album,
PageType::Playlist => MusicPageType::Playlist,
PageType::Channel | PageType::ArtistDiscography => MusicPageType::None,
PageType::Channel => MusicPageType::None,
PageType::Unknown => MusicPageType::Unknown,
}
}
}
impl NavigationEndpoint {
/// Get the YouTube Music page and id from a browse/watch endpoint
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
self.browse_endpoint
.and_then(|be| {
be.browse_endpoint_context_supported_configs.map(|config| {
match self {
NavigationEndpoint::Watch { watch_endpoint } => {
if watch_endpoint
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
Some((MusicPageType::None, watch_endpoint.video_id))
} else {
Some((
MusicPageType::Track {
is_video: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch_endpoint.video_id,
))
}
}
NavigationEndpoint::Browse {
browse_endpoint, ..
} => browse_endpoint
.browse_endpoint_context_supported_configs
.map(|config| {
(
config.browse_endpoint_context_music_config.page_type.into(),
be.browse_id,
browse_endpoint.browse_id,
)
}),
NavigationEndpoint::Url { .. } => None,
}
}
/// Get the page type of a browse endpoint
pub(crate) fn page_type(&self) -> Option<PageType> {
if let NavigationEndpoint::Browse {
browse_endpoint,
command_metadata,
} = self
{
browse_endpoint
.browse_endpoint_context_supported_configs
.as_ref()
.map(|c| c.browse_endpoint_context_music_config.page_type)
.or_else(|| {
command_metadata
.as_ref()
.map(|c| c.web_command_metadata.web_page_type)
})
})
.or_else(|| {
self.watch_endpoint.map(|watch| {
if watch
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
(MusicPageType::None, watch.video_id)
} else {
(
MusicPageType::Track {
is_video: watch
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch.video_id,
)
}
})
})
} else {
None
}
}
/// Get the sanitized URL from a url endpoint
pub(crate) fn url(&self) -> Option<String> {
match self {
NavigationEndpoint::Url { url_endpoint } => {
Some(util::sanitize_yt_url(&url_endpoint.url))
}
_ => None,
}
}
}

6
src/client/response/video_item.rs
View file

@ -732,11 +732,7 @@ impl YouTubeListMapper<YouTubeItem> {
links: meta
.primary_links
.into_iter()
.filter_map(|l| {
l.navigation_endpoint
.url_endpoint
.map(|url| (l.title, util::sanitize_yt_url(&url.url)))
})
.filter_map(|l| l.navigation_endpoint.url().map(|url| (l.title, url)))
.collect(),
});
}

38
src/client/url_resolver.rs
View file

@ -10,7 +10,10 @@ use crate::{
util,
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
use super::{
response::{self, url_endpoint::NavigationEndpoint},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
@ -326,26 +329,21 @@ impl MapResponse<UrlTarget> for response::ResolvedUrl {
_lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
) -> Result<MapResult<UrlTarget>, ExtractionError> {
let browse_endpoint = self
.endpoint
.browse_endpoint
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No browse ID")))?;
let pt = self.endpoint.page_type();
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = self.endpoint
{
let target = pt
.and_then(|pt| pt.to_url_target(browse_endpoint.browse_id))
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?;
let target = self
.endpoint
.command_metadata
.map(|c| c.web_command_metadata.web_page_type)
.or_else(|| {
browse_endpoint
.browse_endpoint_context_supported_configs
.map(|c| c.browse_endpoint_context_music_config.page_type)
Ok(MapResult {
c: target,
warnings: Vec::new(),
})
.and_then(|pt| pt.to_url_target(browse_endpoint.browse_id))
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?;
Ok(MapResult {
c: target,
warnings: Vec::new(),
})
} else {
Err(ExtractionError::InvalidData(Cow::Borrowed("No browse ID")))
}
}
}