Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: increase default timeout to 20s, use ffmpeg copy arg

Browse source 
refactor: use enum for navigation endpoint
This commit is contained in:
ThetaDev 2023-05-17 14:35:10 +02:00
parent dc7bd7befc
commit 805cc5088f
11 changed files with 226 additions and 157 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/main.rs
View file

@ -385,7 +385,7 @@ async fn download_videos(
#[tokio::main]
async fn main() {
env_logger::init();
env_logger::builder().format_timestamp_micros().init();
let cli = Cli::parse();

9
downloader/src/lib.rs
View file

@ -429,7 +429,7 @@ async fn download_streams(
}
async fn convert_streams<P: Into<PathBuf>>(
downloads: &Vec<StreamDownload>,
downloads: &[StreamDownload],
output: P,
ffmpeg: &str,
) -> Result<()> {
@ -448,11 +448,8 @@ async fn convert_streams<P: Into<PathBuf>>(
args.append(&mut mapping_args);
// Combining multiple streams, keep codecs
if downloads.len() > 1 {
args.push("-c".into());
args.push("copy".into());
}
args.push("-c".into());
args.push("copy".into());
args.push(output_path.into());

4
src/client/mod.rs
View file

@ -454,7 +454,7 @@ impl RustyPipeBuilder {
.brotli(true)
.redirect(reqwest::redirect::Policy::none());
if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(10)) {
if let Some(timeout) = self.timeout.or_default(|| Duration::from_secs(20)) {
client_builder = client_builder.timeout(timeout);
}
@ -556,7 +556,7 @@ impl RustyPipeBuilder {
/// The timeout is applied from when the request starts connecting until the
/// response body has finished.
///
/// **Default value**: 10s
/// **Default value**: 20s
#[must_use]
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = DefaultOpt::Some(timeout);

56
src/client/music_artist.rs
View file

@ -4,6 +4,7 @@ use once_cell::sync::Lazy;
use regex::Regex;
use crate::{
client::response::url_endpoint::{MusicPageType, NavigationEndpoint},
error::{Error, ExtractionError},
model::{AlbumItem, ArtistId, MusicArtist},
serializer::MapResult,
@ -188,38 +189,29 @@ fn map_artist_page(
.music_carousel_shelf_basic_header_renderer
.more_content_button
{
if let Some(bep) =
button.button_renderer.navigation_endpoint.browse_endpoint
{
if let Some(cfg) = bep.browse_endpoint_context_supported_configs {
match cfg.browse_endpoint_context_music_config.page_type {
// Music videos
PageType::Playlist => {
if videos_playlist_id.is_none() {
videos_playlist_id = Some(bep.browse_id);
}
}
// Albums
PageType::ArtistDiscography => {
match button.button_renderer.navigation_endpoint.music_page() {
// Music videos
Some((MusicPageType::Playlist, id)) => {
if videos_playlist_id.is_none() {
videos_playlist_id = Some(id);
}
}
// Albums
Some((MusicPageType::ArtistDiscography, _)) => {
can_fetch_more = true;
extendable_albums = true;
}
// Albums or playlists
Some((MusicPageType::Artist, _)) => {
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.page_type() {
can_fetch_more = true;
extendable_albums = true;
}
// Albums or playlists
PageType::Artist => {
// Peek at the first item to determine type
if let Some(response::music_item::MusicResponseItem::MusicTwoRowItemRenderer(item)) = shelf.contents.c.first() {
if let Some(PageType::Album) = item.navigation_endpoint.browse_endpoint.as_ref().and_then(|be| {
be.browse_endpoint_context_supported_configs.as_ref().map(|config| {
config.browse_endpoint_context_music_config.page_type
})}) {
can_fetch_more = true;
extendable_albums = true;
}
}
}
_ => {}
}
}
_ => {}
}
}
}
@ -251,10 +243,12 @@ fn map_artist_page(
});
let radio_id = header.start_radio_button.and_then(|b| {
b.button_renderer
.navigation_endpoint
.watch_endpoint
.and_then(|w| w.playlist_id)
if let NavigationEndpoint::Watch { watch_endpoint } = b.button_renderer.navigation_endpoint
{
watch_endpoint.playlist_id
} else {
None
}
});
Ok(MapResult {

28
src/client/music_genres.rs
View file

@ -7,7 +7,7 @@ use crate::{
};
use super::{
response::{self, music_item::MusicListMapper},
response::{self, music_item::MusicListMapper, url_endpoint::NavigationEndpoint},
ClientType, MapResponse, QBrowse, QBrowseParams, RustyPipeQuery,
};
@ -144,18 +144,20 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
h.music_carousel_shelf_basic_header_renderer
.more_content_button
.and_then(|btn| {
btn.button_renderer
.navigation_endpoint
.browse_endpoint
.and_then(|browse| {
if browse.browse_id
== "FEmusic_moods_and_genres_category"
{
Some(browse.params)
} else {
None
}
})
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = btn.button_renderer.navigation_endpoint
{
if browse_endpoint.browse_id
== "FEmusic_moods_and_genres_category"
{
Some(browse_endpoint.params)
} else {
None
}
} else {
None
}
})
}),
shelf.contents,

24
src/client/response/music_item.rs
View file

@ -759,7 +759,7 @@ impl MusicListMapper {
}));
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => {
MusicPageType::None | MusicPageType::ArtistDiscography => {
// There may be broken YT channels from the artist search. They can be skipped.
Ok(None)
}
@ -901,7 +901,7 @@ impl MusicListMapper {
}));
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => Ok(None),
MusicPageType::None | MusicPageType::ArtistDiscography => Ok(None),
MusicPageType::Unknown => {
self.has_unknown = true;
Ok(None)
@ -1039,7 +1039,7 @@ impl MusicListMapper {
}));
Some(MusicItemType::Playlist)
}
MusicPageType::None => None,
MusicPageType::None | MusicPageType::ArtistDiscography => None,
MusicPageType::Unknown => {
self.has_unknown = true;
None
@ -1171,20 +1171,22 @@ fn map_artist_id_fallback(
pub(crate) fn map_artist_id(entries: Vec<MusicItemMenuEntry>) -> Option<String> {
entries.into_iter().find_map(|i| {
let ep = i
.menu_navigation_item_renderer
.navigation_endpoint
.browse_endpoint;
ep.and_then(|ep| {
ep.browse_endpoint_context_supported_configs
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = i.menu_navigation_item_renderer.navigation_endpoint
{
browse_endpoint
.browse_endpoint_context_supported_configs
.and_then(|cfg| {
if cfg.browse_endpoint_context_music_config.page_type == PageType::Artist {
Some(ep.browse_id)
Some(browse_endpoint.browse_id)
} else {
None
}
})
})
} else {
None
}
})
}

123
src/client/response/url_endpoint.rs
View file

@ -11,21 +11,20 @@ pub(crate) struct ResolvedUrl {
}
#[serde_as]
#[derive(Debug, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub(crate) struct NavigationEndpoint {
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub watch_endpoint: Option<WatchEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub browse_endpoint: Option<BrowseEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub url_endpoint: Option<UrlEndpoint>,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
pub command_metadata: Option<CommandMetadata>,
#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub(crate) enum NavigationEndpoint {
#[serde(rename_all = "camelCase")]
Watch { watch_endpoint: WatchEndpoint },
#[serde(rename_all = "camelCase")]
Browse {
browse_endpoint: BrowseEndpoint,
#[serde(default)]
#[serde_as(deserialize_as = "DefaultOnError")]
command_metadata: Option<CommandMetadata>,
},
#[serde(rename_all = "camelCase")]
Url { url_endpoint: UrlEndpoint },
}
#[derive(Debug, Deserialize)]
@ -184,6 +183,7 @@ impl PageType {
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub(crate) enum MusicPageType {
Artist,
ArtistDiscography,
Album,
Playlist,
Track { is_video: bool },
@ -195,47 +195,82 @@ impl From<PageType> for MusicPageType {
fn from(t: PageType) -> Self {
match t {
PageType::Artist => MusicPageType::Artist,
PageType::ArtistDiscography => MusicPageType::ArtistDiscography,
PageType::Album => MusicPageType::Album,
PageType::Playlist => MusicPageType::Playlist,
PageType::Channel | PageType::ArtistDiscography => MusicPageType::None,
PageType::Channel => MusicPageType::None,
PageType::Unknown => MusicPageType::Unknown,
}
}
}
impl NavigationEndpoint {
/// Get the YouTube Music page and id from a browse/watch endpoint
pub(crate) fn music_page(self) -> Option<(MusicPageType, String)> {
self.browse_endpoint
.and_then(|be| {
be.browse_endpoint_context_supported_configs.map(|config| {
match self {
NavigationEndpoint::Watch { watch_endpoint } => {
if watch_endpoint
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
Some((MusicPageType::None, watch_endpoint.video_id))
} else {
Some((
MusicPageType::Track {
is_video: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch_endpoint.video_id,
))
}
}
NavigationEndpoint::Browse {
browse_endpoint, ..
} => browse_endpoint
.browse_endpoint_context_supported_configs
.map(|config| {
(
config.browse_endpoint_context_music_config.page_type.into(),
be.browse_id,
browse_endpoint.browse_id,
)
}),
NavigationEndpoint::Url { .. } => None,
}
}
/// Get the page type of a browse endpoint
pub(crate) fn page_type(&self) -> Option<PageType> {
if let NavigationEndpoint::Browse {
browse_endpoint,
command_metadata,
} = self
{
browse_endpoint
.browse_endpoint_context_supported_configs
.as_ref()
.map(|c| c.browse_endpoint_context_music_config.page_type)
.or_else(|| {
command_metadata
.as_ref()
.map(|c| c.web_command_metadata.web_page_type)
})
})
.or_else(|| {
self.watch_endpoint.map(|watch| {
if watch
.playlist_id
.map(|plid| plid.starts_with("RDQM"))
.unwrap_or_default()
{
// Genre radios (e.g. "pop radio") will be skipped
(MusicPageType::None, watch.video_id)
} else {
(
MusicPageType::Track {
is_video: watch
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
watch.video_id,
)
}
})
})
} else {
None
}
}
/// Get the sanitized URL from a url endpoint
pub(crate) fn url(&self) -> Option<String> {
match self {
NavigationEndpoint::Url { url_endpoint } => {
Some(util::sanitize_yt_url(&url_endpoint.url))
}
_ => None,
}
}
}

6
src/client/response/video_item.rs
View file

@ -732,11 +732,7 @@ impl YouTubeListMapper<YouTubeItem> {
links: meta
.primary_links
.into_iter()
.filter_map(|l| {
l.navigation_endpoint
.url_endpoint
.map(|url| (l.title, util::sanitize_yt_url(&url.url)))
})
.filter_map(|l| l.navigation_endpoint.url().map(|url| (l.title, url)))
.collect(),
});
}

38
src/client/url_resolver.rs
View file

@ -10,7 +10,10 @@ use crate::{
util,
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
use super::{
response::{self, url_endpoint::NavigationEndpoint},
ClientType, MapResponse, RustyPipeQuery, YTContext,
};
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
@ -326,26 +329,21 @@ impl MapResponse<UrlTarget> for response::ResolvedUrl {
_lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
) -> Result<MapResult<UrlTarget>, ExtractionError> {
let browse_endpoint = self
.endpoint
.browse_endpoint
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No browse ID")))?;
let pt = self.endpoint.page_type();
if let NavigationEndpoint::Browse {
browse_endpoint, ..
} = self.endpoint
{
let target = pt
.and_then(|pt| pt.to_url_target(browse_endpoint.browse_id))
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?;
let target = self
.endpoint
.command_metadata
.map(|c| c.web_command_metadata.web_page_type)
.or_else(|| {
browse_endpoint
.browse_endpoint_context_supported_configs
.map(|c| c.browse_endpoint_context_music_config.page_type)
Ok(MapResult {
c: target,
warnings: Vec::new(),
})
.and_then(|pt| pt.to_url_target(browse_endpoint.browse_id))
.ok_or(ExtractionError::InvalidData(Cow::Borrowed("No page type")))?;
Ok(MapResult {
c: target,
warnings: Vec::new(),
})
} else {
Err(ExtractionError::InvalidData(Cow::Borrowed("No browse ID")))
}
}
}

42
src/model/convert.rs
View file

@ -37,6 +37,12 @@ impl FromYtItem for VideoItem {
}
}
impl From<VideoItem> for YouTubeItem {
fn from(value: VideoItem) -> Self {
Self::Video(value)
}
}
impl FromYtItem for PlaylistItem {
fn from_yt_item(item: YouTubeItem) -> Option<Self> {
match item {
@ -46,6 +52,12 @@ impl FromYtItem for PlaylistItem {
}
}
impl From<PlaylistItem> for YouTubeItem {
fn from(value: PlaylistItem) -> Self {
Self::Playlist(value)
}
}
impl FromYtItem for ChannelItem {
fn from_yt_item(item: YouTubeItem) -> Option<Self> {
match item {
@ -55,6 +67,12 @@ impl FromYtItem for ChannelItem {
}
}
impl From<ChannelItem> for YouTubeItem {
fn from(value: ChannelItem) -> Self {
Self::Channel(value)
}
}
impl FromYtItem for MusicItem {
fn from_ytm_item(item: MusicItem) -> Option<Self> {
Some(item)
@ -70,6 +88,12 @@ impl FromYtItem for TrackItem {
}
}
impl From<TrackItem> for MusicItem {
fn from(value: TrackItem) -> Self {
Self::Track(value)
}
}
impl FromYtItem for AlbumItem {
fn from_ytm_item(item: MusicItem) -> Option<Self> {
match item {
@ -79,6 +103,12 @@ impl FromYtItem for AlbumItem {
}
}
impl From<AlbumItem> for MusicItem {
fn from(value: AlbumItem) -> Self {
Self::Album(value)
}
}
impl FromYtItem for ArtistItem {
fn from_ytm_item(item: MusicItem) -> Option<Self> {
match item {
@ -88,6 +118,12 @@ impl FromYtItem for ArtistItem {
}
}
impl From<ArtistItem> for MusicItem {
fn from(value: ArtistItem) -> Self {
Self::Artist(value)
}
}
impl FromYtItem for MusicPlaylistItem {
fn from_ytm_item(item: MusicItem) -> Option<Self> {
match item {
@ -97,6 +133,12 @@ impl FromYtItem for MusicPlaylistItem {
}
}
impl From<MusicPlaylistItem> for MusicItem {
fn from(value: MusicPlaylistItem) -> Self {
Self::Playlist(value)
}
}
impl<T> From<Channel<T>> for ChannelTag {
fn from(channel: Channel<T>) -> Self {
Self {

51
src/serializer/text.rs
View file

@ -3,7 +3,7 @@ use std::convert::TryFrom;
use once_cell::sync::Lazy;
use regex::Regex;
use serde::{Deserialize, Deserializer};
use serde_with::{serde_as, DeserializeAs};
use serde_with::{serde_as, DeserializeAs, VecSkipError};
use crate::{
client::response::url_endpoint::{MusicVideoType, NavigationEndpoint, PageType},
@ -124,18 +124,19 @@ struct RichTextInternal {
#[serde(rename_all = "camelCase")]
struct RichTextRun {
text: String,
#[serde(default)]
navigation_endpoint: NavigationEndpoint,
navigation_endpoint: Option<NavigationEndpoint>,
}
/// This is a new rich text representation format that YouTube is A/B testing
/// at the moment. It consists of the full text and an array of ranges describing
/// the links.
#[serde_as]
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct AttributedText {
content: String,
#[serde(default)]
#[serde_as(as = "VecSkipError<_>")]
command_runs: Vec<AttributedTextRun>,
}
@ -160,35 +161,37 @@ impl From<RichTextRun> for TextComponent {
}
/// Map a single component of a rich text
fn map_text_component(text: String, nav: NavigationEndpoint) -> TextComponent {
match nav.watch_endpoint {
Some(w) => TextComponent::Video {
fn map_text_component(text: String, nav: Option<NavigationEndpoint>) -> TextComponent {
match nav {
Some(NavigationEndpoint::Watch { watch_endpoint }) => TextComponent::Video {
text,
video_id: w.video_id,
start_time: w.start_time_seconds,
is_video: w
video_id: watch_endpoint.video_id,
start_time: watch_endpoint.start_time_seconds,
is_video: watch_endpoint
.watch_endpoint_music_supported_configs
.watch_endpoint_music_config
.music_video_type
== MusicVideoType::Video,
},
None => match nav.browse_endpoint {
Some(b) => TextComponent::Browse {
page_type: match &b.browse_endpoint_context_supported_configs {
Some(bc) => bc.browse_endpoint_context_music_config.page_type,
None => match &nav.command_metadata {
Some(cm) => cm.web_command_metadata.web_page_type,
None => return TextComponent::Text { text },
},
Some(NavigationEndpoint::Browse {
browse_endpoint,
command_metadata,
}) => TextComponent::Browse {
page_type: match &browse_endpoint.browse_endpoint_context_supported_configs {
Some(bc) => bc.browse_endpoint_context_music_config.page_type,
None => match &command_metadata {
Some(cm) => cm.web_command_metadata.web_page_type,
None => return TextComponent::Text { text },
},
text,
browse_id: b.browse_id,
},
None => match nav.url_endpoint {
Some(u) => TextComponent::Web { text, url: u.url },
None => TextComponent::Text { text },
},
text,
browse_id: browse_endpoint.browse_id,
},
Some(NavigationEndpoint::Url { url_endpoint }) => TextComponent::Web {
text,
url: url_endpoint.url,
},
None => TextComponent::Text { text },
}
}
@ -278,7 +281,7 @@ impl<'de> DeserializeAs<'de, TextComponents> for AttributedText {
}
components.push(map_text_component(
txt_link.to_string(),
cmd.on_tap.innertube_command,
Some(cmd.on_tap.innertube_command),
));
});