diff --git a/cli/src/main.rs b/cli/src/main.rs index 2a1ba78..0a670ef 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -385,7 +385,7 @@ async fn download_videos( #[tokio::main] async fn main() { - env_logger::init(); + env_logger::builder().format_timestamp_micros().init(); let cli = Cli::parse(); diff --git a/downloader/src/lib.rs b/downloader/src/lib.rs index 228d5de..0ad54d5 100644 --- a/downloader/src/lib.rs +++ b/downloader/src/lib.rs @@ -429,7 +429,7 @@ async fn download_streams( } async fn convert_streams>( - downloads: &Vec, + downloads: &[StreamDownload], output: P, ffmpeg: &str, ) -> Result<()> { @@ -448,11 +448,8 @@ async fn convert_streams>( args.append(&mut mapping_args); - // Combining multiple streams, keep codecs - if downloads.len() > 1 { - args.push("-c".into()); - args.push("copy".into()); - } + args.push("-c".into()); + args.push("copy".into()); args.push(output_path.into()); diff --git a/src/client/mod.rs b/src/client/mod.rs index ae7e180..5b9398c 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -454,7 +454,7 @@ impl RustyPipeBuilder { .brotli(true) .redirect(reqwest::redirect::Policy::none()); - if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(10)) { + if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(20)) { client_builder = client_builder.timeout(timeout); } @@ -556,7 +556,7 @@ impl RustyPipeBuilder { /// The timeout is applied from when the request starts connecting until the /// response body has finished. /// - /// **Default value**: 10s + /// **Default value**: 20s #[must_use] pub fn timeout(mut self, timeout: Duration) -> Self { self.timeout = DefaultOpt::Some(timeout); diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 58b926d..d5f2c64 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -4,6 +4,7 @@ use once_cell::sync::Lazy; use regex::Regex; use crate::{ + client::response::url_endpoint::{MusicPageType, NavigationEndpoint}, error::{Error, ExtractionError}, model::{AlbumItem, ArtistId, MusicArtist}, serializer::MapResult, @@ -188,38 +189,29 @@ fn map_artist_page( .music_carousel_shelf_basic_header_renderer .more_content_button { - if let Some(bep) = - button.button_renderer.navigation_endpoint.browse_endpoint - { - if let Some(cfg) = bep.browse_endpoint_context_supported_configs { - match cfg.browse_endpoint_context_music_config.page_type { - // Music videos - PageType::Playlist => { - if videos_playlist_id.is_none() { - videos_playlist_id = Some(bep.browse_id); - } - } - // Albums - PageType::ArtistDiscography => { + match button.button_renderer.navigation_endpoint.music_page() { + // Music videos + Some((MusicPageType::Playlist, id)) => { + if videos_playlist_id.is_none() { + videos_playlist_id = Some(id); + } + } + // Albums + Some((MusicPageType::ArtistDiscography, _)) => { + can_fetch_more = true; + extendable_albums = true; + } + // Albums or playlists + Some((MusicPageType::Artist, _)) => { + // Peek at the first item to determine type + if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() { + if let Some(PageType::Album) = item.navigation_endpoint.page_type() { can_fetch_more = true; extendable_albums = true; } - // Albums or playlists - PageType::Artist => { - // Peek at the first item to determine type - if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() { - if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| { - be.browse_endpoint_context_supported_configs.as_ref().map(|config| { - config.browse_endpoint_context_music_config.page_type - })}) { - can_fetch_more = true; - extendable_albums = true; - } - } - } - _ => {} } } + _ => {} } } } @@ -251,10 +243,12 @@ fn map_artist_page( }); let radio_id = header.start_radio_button.and_then(|b| { - b.button_renderer - .navigation_endpoint - .watch_endpoint - .and_then(|w| w.playlist_id) + if let NavigationEndpoint::Watch { watch_endpoint } = b.button_renderer.navigation_endpoint + { + watch_endpoint.playlist_id + } else { + None + } }); Ok(MapResult { diff --git a/src/client/music_genres.rs b/src/client/music_genres.rs index f471f87..3a696ca 100644 --- a/src/client/music_genres.rs +++ b/src/client/music_genres.rs @@ -7,7 +7,7 @@ use crate::{ }; use super::{ - response::{self, music_item::MusicListMapper}, + response::{self, music_item::MusicListMapper, url_endpoint::NavigationEndpoint}, ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery, }; @@ -144,18 +144,20 @@ impl MapResponse for response::MusicGenre { h.music_carousel_shelf_basic_header_renderer .more_content_button .and_then(|btn| { - btn.button_renderer - .navigation_endpoint - .browse_endpoint - .and_then(|browse| { - if browse.browse_id - == "FEmusic_moods_and_genres_category" - { - Some(browse.params) - } else { - None - } - }) + if let NavigationEndpoint::Browse { + browse_endpoint, .. + } = btn.button_renderer.navigation_endpoint + { + if browse_endpoint.browse_id + == "FEmusic_moods_and_genres_category" + { + Some(browse_endpoint.params) + } else { + None + } + } else { + None + } }) }), shelf.contents, diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index 3edd74e..bee91c9 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -759,7 +759,7 @@ impl MusicListMapper { })); Ok(Some(MusicItemType::Playlist)) } - MusicPageType::None => { + MusicPageType::None | MusicPageType::ArtistDiscography => { // There may be broken YT channels from the artist search. They can be skipped. Ok(None) } @@ -901,7 +901,7 @@ impl MusicListMapper { })); Ok(Some(MusicItemType::Playlist)) } - MusicPageType::None => Ok(None), + MusicPageType::None | MusicPageType::ArtistDiscography => Ok(None), MusicPageType::Unknown => { self.has_unknown = true; Ok(None) @@ -1039,7 +1039,7 @@ impl MusicListMapper { })); Some(MusicItemType::Playlist) } - MusicPageType::None => None, + MusicPageType::None | MusicPageType::ArtistDiscography => None, MusicPageType::Unknown => { self.has_unknown = true; None @@ -1171,20 +1171,22 @@ fn map_artist_id_fallback( pub(crate) fn map_artist_id(entries: Vec) -> Option { entries.into_iter().find_map(|i| { - let ep = i - .menu_navigation_item_renderer - .navigation_endpoint - .browse_endpoint; - ep.and_then(|ep| { - ep.browse_endpoint_context_supported_configs + if let NavigationEndpoint::Browse { + browse_endpoint, .. + } = i.menu_navigation_item_renderer.navigation_endpoint + { + browse_endpoint + .browse_endpoint_context_supported_configs .and_then(|cfg| { if cfg.browse_endpoint_context_music_config.page_type == PageType::Artist { - Some(ep.browse_id) + Some(browse_endpoint.browse_id) } else { None } }) - }) + } else { + None + } }) } diff --git a/src/client/response/url_endpoint.rs b/src/client/response/url_endpoint.rs index 491acdd..1b9b225 100644 --- a/src/client/response/url_endpoint.rs +++ b/src/client/response/url_endpoint.rs @@ -11,21 +11,20 @@ pub(crate) struct ResolvedUrl { } #[serde_as] -#[derive(Debug, Deserialize, Default)] -#[serde(rename_all = "camelCase")] -pub(crate) struct NavigationEndpoint { - #[serde(default)] - #[serde_as(deserialize_as = "DefaultOnError")] - pub watch_endpoint: Option, - #[serde(default)] - #[serde_as(deserialize_as = "DefaultOnError")] - pub browse_endpoint: Option, - #[serde(default)] - #[serde_as(deserialize_as = "DefaultOnError")] - pub url_endpoint: Option, - #[serde(default)] - #[serde_as(deserialize_as = "DefaultOnError")] - pub command_metadata: Option, +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub(crate) enum NavigationEndpoint { + #[serde(rename_all = "camelCase")] + Watch { watch_endpoint: WatchEndpoint }, + #[serde(rename_all = "camelCase")] + Browse { + browse_endpoint: BrowseEndpoint, + #[serde(default)] + #[serde_as(deserialize_as = "DefaultOnError")] + command_metadata: Option, + }, + #[serde(rename_all = "camelCase")] + Url { url_endpoint: UrlEndpoint }, } #[derive(Debug, Deserialize)] @@ -184,6 +183,7 @@ impl PageType { #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum MusicPageType { Artist, + ArtistDiscography, Album, Playlist, Track { is_video: bool }, @@ -195,47 +195,82 @@ impl From for MusicPageType { fn from(t: PageType) -> Self { match t { PageType::Artist => MusicPageType::Artist, + PageType::ArtistDiscography => MusicPageType::ArtistDiscography, PageType::Album => MusicPageType::Album, PageType::Playlist => MusicPageType::Playlist, - PageType::Channel | PageType::ArtistDiscography => MusicPageType::None, + PageType::Channel => MusicPageType::None, PageType::Unknown => MusicPageType::Unknown, } } } impl NavigationEndpoint { + /// Get the YouTube Music page and id from a browse/watch endpoint pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> { - self.browse_endpoint - .and_then(|be| { - be.browse_endpoint_context_supported_configs.map(|config| { + match self { + NavigationEndpoint::Watch { watch_endpoint } => { + if watch_endpoint + .playlist_id + .map(|plid| plid.starts_with("RDQM")) + .unwrap_or_default() + { + // Genre radios (e.g. "pop radio") will be skipped + Some((MusicPageType::None, watch_endpoint.video_id)) + } else { + Some(( + MusicPageType::Track { + is_video: watch_endpoint + .watch_endpoint_music_supported_configs + .watch_endpoint_music_config + .music_video_type + == MusicVideoType::Video, + }, + watch_endpoint.video_id, + )) + } + } + NavigationEndpoint::Browse { + browse_endpoint, .. + } => browse_endpoint + .browse_endpoint_context_supported_configs + .map(|config| { ( config.browse_endpoint_context_music_config.page_type.into(), - be.browse_id, + browse_endpoint.browse_id, ) + }), + NavigationEndpoint::Url { .. } => None, + } + } + + /// Get the page type of a browse endpoint + pub(crate) fn page_type(&self) -> Option { + if let NavigationEndpoint::Browse { + browse_endpoint, + command_metadata, + } = self + { + browse_endpoint + .browse_endpoint_context_supported_configs + .as_ref() + .map(|c| c.browse_endpoint_context_music_config.page_type) + .or_else(|| { + command_metadata + .as_ref() + .map(|c| c.web_command_metadata.web_page_type) }) - }) - .or_else(|| { - self.watch_endpoint.map(|watch| { - if watch - .playlist_id - .map(|plid| plid.starts_with("RDQM")) - .unwrap_or_default() - { - // Genre radios (e.g. "pop radio") will be skipped - (MusicPageType::None, watch.video_id) - } else { - ( - MusicPageType::Track { - is_video: watch - .watch_endpoint_music_supported_configs - .watch_endpoint_music_config - .music_video_type - == MusicVideoType::Video, - }, - watch.video_id, - ) - } - }) - }) + } else { + None + } + } + + /// Get the sanitized URL from a url endpoint + pub(crate) fn url(&self) -> Option { + match self { + NavigationEndpoint::Url { url_endpoint } => { + Some(util::sanitize_yt_url(&url_endpoint.url)) + } + _ => None, + } } } diff --git a/src/client/response/video_item.rs b/src/client/response/video_item.rs index e13f2a3..fa0577c 100644 --- a/src/client/response/video_item.rs +++ b/src/client/response/video_item.rs @@ -732,11 +732,7 @@ impl YouTubeListMapper { links: meta .primary_links .into_iter() - .filter_map(|l| { - l.navigation_endpoint - .url_endpoint - .map(|url| (l.title, util::sanitize_yt_url(&url.url))) - }) + .filter_map(|l| l.navigation_endpoint.url().map(|url| (l.title, url))) .collect(), }); } diff --git a/src/client/url_resolver.rs b/src/client/url_resolver.rs index c001a2b..6b9e83b 100644 --- a/src/client/url_resolver.rs +++ b/src/client/url_resolver.rs @@ -10,7 +10,10 @@ use crate::{ util, }; -use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; +use super::{ + response::{self, url_endpoint::NavigationEndpoint}, + ClientType, MapResponse, RustyPipeQuery, YTContext, +}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -326,26 +329,21 @@ impl MapResponse for response::ResolvedUrl { _lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, ) -> Result, ExtractionError> { - let browse_endpoint = self - .endpoint - .browse_endpoint - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("No browse ID")))?; + let pt = self.endpoint.page_type(); + if let NavigationEndpoint::Browse { + browse_endpoint, .. + } = self.endpoint + { + let target = pt + .and_then(|pt| pt.to_url_target(browse_endpoint.browse_id)) + .ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?; - let target = self - .endpoint - .command_metadata - .map(|c| c.web_command_metadata.web_page_type) - .or_else(|| { - browse_endpoint - .browse_endpoint_context_supported_configs - .map(|c| c.browse_endpoint_context_music_config.page_type) + Ok(MapResult { + c: target, + warnings: Vec::new(), }) - .and_then(|pt| pt.to_url_target(browse_endpoint.browse_id)) - .ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?; - - Ok(MapResult { - c: target, - warnings: Vec::new(), - }) + } else { + Err(ExtractionError::InvalidData(Cow::Borrowed("No browse ID"))) + } } } diff --git a/src/model/convert.rs b/src/model/convert.rs index 6e2c71c..0800006 100644 --- a/src/model/convert.rs +++ b/src/model/convert.rs @@ -37,6 +37,12 @@ impl FromYtItem for VideoItem { } } +impl From for YouTubeItem { + fn from(value: VideoItem) -> Self { + Self::Video(value) + } +} + impl FromYtItem for PlaylistItem { fn from_yt_item(item: YouTubeItem) -> Option { match item { @@ -46,6 +52,12 @@ impl FromYtItem for PlaylistItem { } } +impl From for YouTubeItem { + fn from(value: PlaylistItem) -> Self { + Self::Playlist(value) + } +} + impl FromYtItem for ChannelItem { fn from_yt_item(item: YouTubeItem) -> Option { match item { @@ -55,6 +67,12 @@ impl FromYtItem for ChannelItem { } } +impl From for YouTubeItem { + fn from(value: ChannelItem) -> Self { + Self::Channel(value) + } +} + impl FromYtItem for MusicItem { fn from_ytm_item(item: MusicItem) -> Option { Some(item) @@ -70,6 +88,12 @@ impl FromYtItem for TrackItem { } } +impl From for MusicItem { + fn from(value: TrackItem) -> Self { + Self::Track(value) + } +} + impl FromYtItem for AlbumItem { fn from_ytm_item(item: MusicItem) -> Option { match item { @@ -79,6 +103,12 @@ impl FromYtItem for AlbumItem { } } +impl From for MusicItem { + fn from(value: AlbumItem) -> Self { + Self::Album(value) + } +} + impl FromYtItem for ArtistItem { fn from_ytm_item(item: MusicItem) -> Option { match item { @@ -88,6 +118,12 @@ impl FromYtItem for ArtistItem { } } +impl From for MusicItem { + fn from(value: ArtistItem) -> Self { + Self::Artist(value) + } +} + impl FromYtItem for MusicPlaylistItem { fn from_ytm_item(item: MusicItem) -> Option { match item { @@ -97,6 +133,12 @@ impl FromYtItem for MusicPlaylistItem { } } +impl From for MusicItem { + fn from(value: MusicPlaylistItem) -> Self { + Self::Playlist(value) + } +} + impl From> for ChannelTag { fn from(channel: Channel) -> Self { Self { diff --git a/src/serializer/text.rs b/src/serializer/text.rs index d883ea9..2e5549d 100644 --- a/src/serializer/text.rs +++ b/src/serializer/text.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use once_cell::sync::Lazy; use regex::Regex; use serde::{Deserialize, Deserializer}; -use serde_with::{serde_as, DeserializeAs}; +use serde_with::{serde_as, DeserializeAs, VecSkipError}; use crate::{ client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType}, @@ -124,18 +124,19 @@ struct RichTextInternal { #[serde(rename_all = "camelCase")] struct RichTextRun { text: String, - #[serde(default)] - navigation_endpoint: NavigationEndpoint, + navigation_endpoint: Option, } /// This is a new rich text representation format that YouTube is A/B testing /// at the moment. It consists of the full text and an array of ranges describing /// the links. +#[serde_as] #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct AttributedText { content: String, #[serde(default)] + #[serde_as(as = "VecSkipError<_>")] command_runs: Vec, } @@ -160,35 +161,37 @@ impl From for TextComponent { } /// Map a single component of a rich text -fn map_text_component(text: String, nav: NavigationEndpoint) -> TextComponent { - match nav.watch_endpoint { - Some(w) => TextComponent::Video { +fn map_text_component(text: String, nav: Option) -> TextComponent { + match nav { + Some(NavigationEndpoint::Watch { watch_endpoint }) => TextComponent::Video { text, - video_id: w.video_id, - start_time: w.start_time_seconds, - is_video: w + video_id: watch_endpoint.video_id, + start_time: watch_endpoint.start_time_seconds, + is_video: watch_endpoint .watch_endpoint_music_supported_configs .watch_endpoint_music_config .music_video_type == MusicVideoType::Video, }, - None => match nav.browse_endpoint { - Some(b) => TextComponent::Browse { - page_type: match &b.browse_endpoint_context_supported_configs { - Some(bc) => bc.browse_endpoint_context_music_config.page_type, - None => match &nav.command_metadata { - Some(cm) => cm.web_command_metadata.web_page_type, - None => return TextComponent::Text { text }, - }, + Some(NavigationEndpoint::Browse { + browse_endpoint, + command_metadata, + }) => TextComponent::Browse { + page_type: match &browse_endpoint.browse_endpoint_context_supported_configs { + Some(bc) => bc.browse_endpoint_context_music_config.page_type, + None => match &command_metadata { + Some(cm) => cm.web_command_metadata.web_page_type, + None => return TextComponent::Text { text }, }, - text, - browse_id: b.browse_id, - }, - None => match nav.url_endpoint { - Some(u) => TextComponent::Web { text, url: u.url }, - None => TextComponent::Text { text }, }, + text, + browse_id: browse_endpoint.browse_id, }, + Some(NavigationEndpoint::Url { url_endpoint }) => TextComponent::Web { + text, + url: url_endpoint.url, + }, + None => TextComponent::Text { text }, } } @@ -278,7 +281,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText { } components.push(map_text_component( txt_link.to_string(), - cmd.on_tap.innertube_command, + Some(cmd.on_tap.innertube_command), )); });