From f526ab38eb72b71353d104647d43cb5f1e40a0f7 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Fri, 9 Dec 2022 01:01:25 +0100 Subject: [PATCH] refactor!: refactored response models doc: documented all public methods --- Cargo.toml | 1 + codegen/src/abtest.rs | 4 +- codegen/src/download_testfiles.rs | 4 +- downloader/src/lib.rs | 2 +- src/cache.rs | 15 +++ src/client/channel.rs | 12 +- src/client/channel_rss.rs | 6 + src/client/mod.rs | 28 +++-- src/client/music_artist.rs | 3 + src/client/music_charts.rs | 1 + src/client/music_details.rs | 16 ++- src/client/music_genres.rs | 2 + src/client/music_new.rs | 4 +- src/client/music_playlist.rs | 8 +- src/client/music_search.rs | 15 ++- src/client/pagination.rs | 25 +++- src/client/player.rs | 4 +- src/client/playlist.rs | 4 +- src/client/response/music_item.rs | 24 ++-- src/client/search.rs | 7 +- src/client/trends.rs | 8 +- src/client/url_resolver.rs | 56 +++++++++ src/client/video_details.rs | 10 +- src/error.rs | 28 +++++ src/lib.rs | 8 +- src/model/convert.rs | 9 ++ src/model/mod.rs | 200 +++++++++++------------------- src/model/ordering.rs | 6 + src/model/paginator.rs | 39 +++++- src/model/richtext.rs | 24 +++- src/model/traits.rs | 130 +++++++++++++++++++ src/param/mod.rs | 35 +----- src/param/search_filter.rs | 34 +++-- src/param/stream_filter.rs | 7 +- src/report.rs | 36 +++++- src/timeago.rs | 10 ++ tests/youtube.rs | 30 ++--- 37 files changed, 600 insertions(+), 255 deletions(-) create mode 100644 src/model/traits.rs diff --git a/Cargo.toml b/Cargo.toml index 1f2c729..44e11fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,5 +56,6 @@ env_logger = "0.10.0" test-log = "0.2.11" rstest = "0.16.0" temp_testdir = "0.2.3" +tokio-test = "0.4.2" insta = { version = "1.17.1", features = ["ron", "redactions"] } path_macro = "1.0.0" diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index 1ec83c6..811a90e 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -4,7 +4,7 @@ use indicatif::{ProgressBar, ProgressStyle}; use num_enum::TryFromPrimitive; use rustypipe::client::{ClientType, RustyPipe, YTContext}; use rustypipe::model::YouTubeItem; -use rustypipe::param::search_filter::{Entity, SearchFilter}; +use rustypipe::param::search_filter::{ItemType, SearchFilter}; use serde::{Deserialize, Serialize}; #[derive( @@ -158,7 +158,7 @@ pub async fn channel_handles_in_search_results(rp: &RustyPipe, visitor_data: &st let search = rp .query() .visitor_data(visitor_data) - .search_filter("rust", &SearchFilter::new().entity(Entity::Channel)) + .search_filter("rust", &SearchFilter::new().item_type(ItemType::Channel)) .await .unwrap(); diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 1e70729..97fe79e 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -8,7 +8,7 @@ use std::{ use rustypipe::{ client::{ClientType, RustyPipe}, param::{ - search_filter::{self, Entity, SearchFilter}, + search_filter::{self, ItemType, SearchFilter}, Country, }, report::{Report, Reporter}, @@ -449,7 +449,7 @@ async fn search_playlists(testfiles: &Path) { let rp = rp_testfile(&json_path); rp.query() - .search_filter("pop", &SearchFilter::new().entity(Entity::Playlist)) + .search_filter("pop", &SearchFilter::new().item_type(ItemType::Playlist)) .await .unwrap(); } diff --git a/downloader/src/lib.rs b/downloader/src/lib.rs index 34adc42..0c8e5ce 100644 --- a/downloader/src/lib.rs +++ b/downloader/src/lib.rs @@ -12,7 +12,7 @@ use rand::Rng; use regex::Regex; use reqwest::{header, Client}; use rustypipe::{ - model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer}, + model::{traits::FileFormat, AudioCodec, VideoCodec, VideoPlayer}, param::StreamFilter, }; use tokio::{ diff --git a/src/cache.rs b/src/cache.rs index fea15a8..9060759 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -7,16 +7,31 @@ use std::{ use log::error; +/// RustyPipe has to cache some information fetched from YouTube: specifically +/// the client versions and the JavaScript code used to deobfuscate the stream URLs. +/// +/// This trait is used to abstract the cache storage behavior so you can store +/// cache data in your preferred way (File, SQL, Redis, etc). +/// +/// The cache is read when building the [`crate::client::RustyPipe`] client and updated +/// whenever additional data is fetched. pub trait CacheStorage: Sync + Send { + /// Write the given string to the cache fn write(&self, data: &str); + /// Read the string from the cache + /// + /// Returns [`None`] when the cache is empty or the reading failed. fn read(&self) -> Option; } +/// [`CacheStorage`] implementation that writes the cache to a JSON file +/// at the given location. pub struct FileStorage { path: PathBuf, } impl FileStorage { + /// Create a new JSON-file based cache storage pub fn new>(path: P) -> Self { Self { path: path.as_ref().to_path_buf(), diff --git a/src/client/channel.rs b/src/client/channel.rs index 4bbac26..eddb15b 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -5,7 +5,7 @@ use url::Url; use crate::{ error::{Error, ExtractionError}, - model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem, YouTubeItem}, + model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem}, param::Language, serializer::MapResult, util, @@ -66,6 +66,7 @@ impl RustyPipeQuery { .await } + /// Get the videos from a YouTube channel pub async fn channel_videos>( &self, channel_id: S, @@ -74,6 +75,7 @@ impl RustyPipeQuery { .await } + /// Get the short videos from a YouTube channel pub async fn channel_shorts>( &self, channel_id: S, @@ -82,6 +84,7 @@ impl RustyPipeQuery { .await } + /// Get the livestreams from a YouTube channel pub async fn channel_livestreams>( &self, channel_id: S, @@ -90,6 +93,7 @@ impl RustyPipeQuery { .await } + /// Search the videos of a channel pub async fn channel_search, S2: AsRef>( &self, channel_id: S, @@ -104,6 +108,7 @@ impl RustyPipeQuery { .await } + /// Get the playlists of a channel pub async fn channel_playlists>( &self, channel_id: S, @@ -127,6 +132,7 @@ impl RustyPipeQuery { .await } + /// Get additional metadata from the *About* tab of a channel pub async fn channel_info>( &self, channel_id: S, @@ -181,7 +187,7 @@ impl MapResponse>> for response::Channel { mapper.items, mapper.ctoken, self.response_context.visitor_data, - crate::param::ContinuationEndpoint::Browse, + crate::model::paginator::ContinuationEndpoint::Browse, ); Ok(MapResult { @@ -487,7 +493,7 @@ mod tests { use crate::{ client::{response, MapResponse}, - model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem}, + model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem}, param::Language, serializer::MapResult, }; diff --git a/src/client/channel_rss.rs b/src/client/channel_rss.rs index 3e5f615..8153755 100644 --- a/src/client/channel_rss.rs +++ b/src/client/channel_rss.rs @@ -9,6 +9,12 @@ use crate::{ use super::{response, RustyPipeQuery}; impl RustyPipeQuery { + /// Get the 15 latest videos from the channel's RSS feed + /// + /// Example: + /// + /// Fetching RSS feeds is a lot faster than querying the InnerTube API, so this method is great + /// for checking a lot of channels or implementing a subscription feed. pub async fn channel_rss>(&self, channel_id: S) -> Result { let url = format!( "https://www.youtube.com/feeds/videos.xml?channel_id={}", diff --git a/src/client/mod.rs b/src/client/mod.rs index aeaa211..e9cb259 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -48,20 +48,26 @@ use crate::{ /// /// There are multiple clients for accessing the YouTube API which have /// slightly different features -/// -/// - **Desktop**: used by youtube.com -/// - **DesktopMusic**: used by music.youtube.com, can access special music data, -/// cannot access non-music content -/// - **TvHtml5Embed**: used by Smart TVs, can access age-restricted videos -/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams -/// - **Ios**: used by the iOS app, no obfuscated URLs #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "snake_case")] pub enum ClientType { + /// Client used by youtube.com Desktop, + /// Client used by music.youtube.com + /// + /// can access YTM-specific data, cannot access non-music content DesktopMusic, + /// used by Smart TVs + /// + /// can access age-restricted videos, cannot access non-embeddable videos TvHtml5Embed, + /// used by the Android app + /// + /// no obfuscated stream URLs, includes lower resolution audio streams Android, + /// used by the iOS app + /// + /// no obfuscated stream URLs Ios, } @@ -74,6 +80,7 @@ impl ClientType { } } +/// YouTube context request parameter #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct YTContext<'a> { @@ -204,6 +211,7 @@ struct RustyPipeOpts { visitor_data: Option, } +/// Builder to construct a new RustyPipe client pub struct RustyPipeBuilder { storage: Option>, reporter: Option>, @@ -212,6 +220,10 @@ pub struct RustyPipeBuilder { default_opts: RustyPipeOpts, } +/// RustyPipe query object +/// +/// Contains a reference to the RustyPipe client as well as query-specific +/// options (e.g. language preference). #[derive(Clone)] pub struct RustyPipeQuery { client: RustyPipe, @@ -257,7 +269,7 @@ enum CacheEntry { } #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ClientData { +struct ClientData { pub version: String, } diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 157c8c2..688f6ea 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -26,6 +26,9 @@ struct QBrowseParams<'a> { } impl RustyPipeQuery { + /// Get a YouTube Music artist page + /// + /// Set `all_albums` to [`true`] if you want to fetch the albums behind the *More* buttons, too. pub async fn music_artist>( &self, artist_id: S, diff --git a/src/client/music_charts.rs b/src/client/music_charts.rs index a812e29..276bf3c 100644 --- a/src/client/music_charts.rs +++ b/src/client/music_charts.rs @@ -31,6 +31,7 @@ struct FormData { } impl RustyPipeQuery { + /// Get the YouTube Music charts for a given country pub async fn music_charts(&self, country: Option) -> Result { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QCharts { diff --git a/src/client/music_details.rs b/src/client/music_details.rs index a93ca5e..40512cd 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -4,7 +4,7 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{ArtistId, Lyrics, MusicRelated, Paginator, TrackDetails, TrackItem}, + model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem}, param::Language, serializer::MapResult, }; @@ -37,6 +37,7 @@ struct QRadio<'a> { } impl RustyPipeQuery { + /// Get the metadata of a YouTube music track pub async fn music_details>(&self, video_id: S) -> Result { let video_id = video_id.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; @@ -58,6 +59,9 @@ impl RustyPipeQuery { .await } + /// Get the lyrics of a YouTube music track + /// + /// The `lyrics_id` has to be obtained using [`RustyPipeQuery::music_details`]. pub async fn music_lyrics>(&self, lyrics_id: S) -> Result { let lyrics_id = lyrics_id.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; @@ -76,6 +80,9 @@ impl RustyPipeQuery { .await } + /// Get related items (tracks, playlists, artists) to a YouTube Music track + /// + /// The `related_id` has to be obtained using [`RustyPipeQuery::music_details`]. pub async fn music_related>(&self, related_id: S) -> Result { let related_id = related_id.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; @@ -94,6 +101,9 @@ impl RustyPipeQuery { .await } + /// Get a YouTube Music radio (a dynamically generated playlist) + /// + /// The `radio_id` can be obtained using [`RustyPipeQuery::music_artist`] to get an artist's radio. pub async fn music_radio>( &self, radio_id: S, @@ -122,6 +132,7 @@ impl RustyPipeQuery { .await } + /// Get a YouTube Music radio (a dynamically generated playlist) for a track pub async fn music_radio_track>( &self, video_id: S, @@ -130,6 +141,7 @@ impl RustyPipeQuery { .await } + /// Get a YouTube Music radio (a dynamically generated playlist) for a playlist pub async fn music_radio_playlist>( &self, playlist_id: S, @@ -263,7 +275,7 @@ impl MapResponse> for response::MusicDetails { tracks, ctoken, None, - crate::param::ContinuationEndpoint::MusicNext, + crate::model::paginator::ContinuationEndpoint::MusicNext, ), warnings: content.contents.warnings, }) diff --git a/src/client/music_genres.rs b/src/client/music_genres.rs index 71d2996..6786a81 100644 --- a/src/client/music_genres.rs +++ b/src/client/music_genres.rs @@ -22,6 +22,7 @@ struct QGenre<'a> { } impl RustyPipeQuery { + /// Get a list of moods and genres from YouTube Music pub async fn music_genres(&self) -> Result, Error> { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { @@ -39,6 +40,7 @@ impl RustyPipeQuery { .await } + /// Get the playlists from a YouTube Music genre pub async fn music_genre>(&self, genre_id: S) -> Result { let genre_id = genre_id.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; diff --git a/src/client/music_new.rs b/src/client/music_new.rs index 1195bf5..bd6725f 100644 --- a/src/client/music_new.rs +++ b/src/client/music_new.rs @@ -3,12 +3,13 @@ use std::borrow::Cow; use crate::{ client::response::music_item::MusicListMapper, error::{Error, ExtractionError}, - model::{AlbumItem, FromYtItem, TrackItem}, + model::{traits::FromYtItem, AlbumItem, TrackItem}, }; use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { + /// Get the new albums that were released on YouTube Music pub async fn music_new_albums(&self) -> Result, Error> { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { @@ -26,6 +27,7 @@ impl RustyPipeQuery { .await } + /// Get the new music videos that were released on YouTube Music pub async fn music_new_videos(&self) -> Result, Error> { let context = self.get_context(ClientType::DesktopMusic, true, None).await; let request_body = QBrowse { diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 92e5e79..5494fab 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, - model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator, TrackItem}, + model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem}, serializer::MapResult, util::{self, TryRemove}, }; @@ -16,6 +16,7 @@ use super::{ }; impl RustyPipeQuery { + /// Get a playlist from YouTube Music pub async fn music_playlist>( &self, playlist_id: S, @@ -37,6 +38,7 @@ impl RustyPipeQuery { .await } + /// Get an album from YouTube Music pub async fn music_album>(&self, album_id: S) -> Result { let album_id = album_id.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; @@ -221,14 +223,14 @@ impl MapResponse for response::MusicPlaylist { map_res.c, ctoken, None, - crate::param::ContinuationEndpoint::MusicBrowse, + crate::model::paginator::ContinuationEndpoint::MusicBrowse, ), related_playlists: Paginator::new_ext( None, Vec::new(), related_ctoken, None, - crate::param::ContinuationEndpoint::MusicBrowse, + crate::model::paginator::ContinuationEndpoint::MusicBrowse, ), }, warnings: map_res.warnings, diff --git a/src/client/music_search.rs b/src/client/music_search.rs index 6981a10..00caa6f 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -6,8 +6,8 @@ use crate::{ client::response::music_item::MusicListMapper, error::{Error, ExtractionError}, model::{ - AlbumItem, ArtistItem, FromYtItem, MusicPlaylistItem, MusicSearchFiltered, - MusicSearchResult, Paginator, TrackItem, + paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem, + MusicSearchFiltered, MusicSearchResult, TrackItem, }, serializer::MapResult, util::TryRemove, @@ -50,6 +50,7 @@ enum Params { } impl RustyPipeQuery { + /// Search YouTube Music. Returns items from any type. pub async fn music_search>(&self, query: S) -> Result { let query = query.as_ref(); let context = self.get_context(ClientType::DesktopMusic, true, None).await; @@ -69,6 +70,7 @@ impl RustyPipeQuery { .await } + /// Search YouTube Music tracks pub async fn music_search_tracks>( &self, query: S, @@ -76,6 +78,7 @@ impl RustyPipeQuery { self._music_search_tracks(query, Params::Tracks).await } + /// Search YouTube Music videos pub async fn music_search_videos>( &self, query: S, @@ -106,6 +109,7 @@ impl RustyPipeQuery { .await } + /// Search YouTube Music albums pub async fn music_search_albums>( &self, query: S, @@ -128,6 +132,7 @@ impl RustyPipeQuery { .await } + /// Search YouTube Music artists pub async fn music_search_artists( &self, query: &str, @@ -149,6 +154,7 @@ impl RustyPipeQuery { .await } + /// Search YouTube Music playlists pub async fn music_search_playlists>( &self, query: S, @@ -156,6 +162,8 @@ impl RustyPipeQuery { self._music_search_playlists(query, Params::Playlists).await } + /// Search YouTube Music playlists that were created by users + /// (`community=true`) or by YouTube Music (`community=false`) pub async fn music_search_playlists_filter>( &self, query: S, @@ -194,6 +202,7 @@ impl RustyPipeQuery { .await } + /// Get YouTube Music search suggestions pub async fn music_search_suggestion>( &self, query: S, @@ -316,7 +325,7 @@ impl MapResponse> for response::MusicSearc map_res.c, ctoken, None, - crate::param::ContinuationEndpoint::MusicSearch, + crate::model::paginator::ContinuationEndpoint::MusicSearch, ), corrected_query, }, diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 8789ad1..ec41838 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -1,8 +1,11 @@ use std::borrow::Cow; use crate::error::{Error, ExtractionError}; -use crate::model::{Comment, FromYtItem, MusicItem, Paginator, PlaylistVideo, YouTubeItem}; -use crate::param::ContinuationEndpoint; +use crate::model::{ + paginator::{ContinuationEndpoint, Paginator}, + traits::FromYtItem, + Comment, MusicItem, PlaylistVideo, YouTubeItem, +}; use crate::serializer::MapResult; use crate::util::TryRemove; @@ -10,6 +13,7 @@ use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanel use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery}; impl RustyPipeQuery { + /// Get more YouTube items from the given continuation token and endpoint pub async fn continuation>( &self, ctoken: S, @@ -174,6 +178,7 @@ impl MapResponse> for response::MusicContinuation { } impl Paginator { + /// Get the next page from the paginator (or `None` if the paginator is exhausted) pub async fn next>(&self, query: Q) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some( @@ -186,6 +191,9 @@ impl Paginator { }) } + /// Extend the items of the paginator by the next page + /// + /// Returns false if the paginator is exhausted. pub async fn extend>(&mut self, query: Q) -> Result { match self.next(query).await { Ok(Some(paginator)) => { @@ -199,6 +207,8 @@ impl Paginator { } } + /// Extend the items of the paginator by the given amount of pages + /// or until the paginator is exhausted. pub async fn extend_pages>( &mut self, query: Q, @@ -215,6 +225,8 @@ impl Paginator { Ok(()) } + /// Extend the items of the paginator until the given amount of items + /// is reached or the paginator is exhausted. pub async fn extend_limit>( &mut self, query: Q, @@ -233,6 +245,7 @@ impl Paginator { } impl Paginator { + /// Get the next page from the paginator (or `None` if the paginator is exhausted) pub async fn next>(&self, query: Q) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some( @@ -247,6 +260,7 @@ impl Paginator { } impl Paginator { + /// Get the next page from the paginator (or `None` if the paginator is exhausted) pub async fn next>(&self, query: Q) -> Result, Error> { Ok(match &self.ctoken { Some(ctoken) => Some(query.as_ref().playlist_continuation(ctoken).await?), @@ -258,6 +272,9 @@ impl Paginator { macro_rules! paginator { ($entity_type:ty) => { impl Paginator<$entity_type> { + /// Extend the items of the paginator by the next page + /// + /// Returns false if the paginator is exhausted. pub async fn extend>( &mut self, query: Q, @@ -274,6 +291,8 @@ macro_rules! paginator { } } + /// Extend the items of the paginator by the given amount of pages + /// or until the paginator is exhausted. pub async fn extend_pages>( &mut self, query: Q, @@ -290,6 +309,8 @@ macro_rules! paginator { Ok(()) } + /// Extend the items of the paginator until the given amount of items + /// is reached or the paginator is exhausted. pub async fn extend_limit>( &mut self, query: Q, diff --git a/src/client/player.rs b/src/client/player.rs index 8262b5d..3072a1a 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -12,7 +12,7 @@ use crate::{ deobfuscate::Deobfuscator, error::{DeobfError, Error, ExtractionError}, model::{ - AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, QualityOrd, Subtitle, + traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Subtitle, VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream, }, param::Language, @@ -58,6 +58,7 @@ struct QContentPlaybackContext { } impl RustyPipeQuery { + /// Get YouTube player data (video/audio streams + basic metadata) pub async fn player>(&self, video_id: S) -> Result { let video_id = video_id.as_ref(); let q1 = self.clone(); @@ -86,6 +87,7 @@ impl RustyPipeQuery { } } + /// Get YouTube player data (video/audio streams + basic metadata) using the specified client pub async fn player_from_client>( &self, video_id: S, diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 98b417b..94d5601 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -5,7 +5,7 @@ use time::OffsetDateTime; use crate::{ deobfuscate::Deobfuscator, error::{Error, ExtractionError}, - model::{ChannelId, Paginator, Playlist, PlaylistVideo}, + model::{paginator::Paginator, ChannelId, Playlist, PlaylistVideo}, param::Language, timeago, util::{self, TryRemove}, @@ -14,6 +14,7 @@ use crate::{ use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery}; impl RustyPipeQuery { + /// Get a YouTube playlist pub async fn playlist>(&self, playlist_id: S) -> Result { let playlist_id = playlist_id.as_ref(); let context = self.get_context(ClientType::Desktop, true, None).await; @@ -32,6 +33,7 @@ impl RustyPipeQuery { .await } + /// Get more playlist items using the given continuation token pub async fn playlist_continuation>( &self, ctoken: S, diff --git a/src/client/response/music_item.rs b/src/client/response/music_item.rs index caffaf3..2e142fc 100644 --- a/src/client/response/music_item.rs +++ b/src/client/response/music_item.rs @@ -3,8 +3,8 @@ use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkip use crate::{ model::{ - self, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, FromYtItem, - MusicEntityType, MusicItem, MusicPlaylistItem, TrackItem, + self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, + MusicItem, MusicItemType, MusicPlaylistItem, TrackItem, }, param::Language, serializer::{ @@ -465,7 +465,7 @@ impl MusicListMapper { } } - fn map_item(&mut self, item: MusicResponseItem) -> Result, String> { + fn map_item(&mut self, item: MusicResponseItem) -> Result, String> { match item { // List item MusicResponseItem::MusicResponsiveListItemRenderer(item) => { @@ -636,7 +636,7 @@ impl MusicListMapper { is_video, track_nr, })); - Ok(Some(MusicEntityType::Track)) + Ok(Some(MusicItemType::Track)) } // Artist / Album / Playlist Some((page_type, id)) => { @@ -666,7 +666,7 @@ impl MusicListMapper { avatar: item.thumbnail.into(), subscriber_count, })); - Ok(Some(MusicEntityType::Artist)) + Ok(Some(MusicItemType::Artist)) } MusicPageType::Album => { let album_type = subtitle_p1 @@ -690,7 +690,7 @@ impl MusicListMapper { year, by_va, })); - Ok(Some(MusicEntityType::Album)) + Ok(Some(MusicItemType::Album)) } MusicPageType::Playlist => { // Part 1 may be the "Playlist" label @@ -717,7 +717,7 @@ impl MusicListMapper { track_count, from_ytm, })); - Ok(Some(MusicEntityType::Playlist)) + Ok(Some(MusicItemType::Playlist)) } MusicPageType::None => { // There may be broken YT channels from the artist search. They can be skipped. @@ -762,7 +762,7 @@ impl MusicListMapper { is_video, track_nr: None, })); - Ok(Some(MusicEntityType::Track)) + Ok(Some(MusicItemType::Track)) } MusicPageType::Artist => { let subscriber_count = subtitle_p1 @@ -774,7 +774,7 @@ impl MusicListMapper { avatar: item.thumbnail_renderer.into(), subscriber_count, })); - Ok(Some(MusicEntityType::Artist)) + Ok(Some(MusicItemType::Artist)) } MusicPageType::Album => { let mut year = None; @@ -818,7 +818,7 @@ impl MusicListMapper { year, by_va, })); - Ok(Some(MusicEntityType::Album)) + Ok(Some(MusicItemType::Album)) } MusicPageType::Playlist => { // When the playlist subtitle has only 1 part, it is a playlist from YT Music @@ -841,7 +841,7 @@ impl MusicListMapper { track_count, from_ytm, })); - Ok(Some(MusicEntityType::Playlist)) + Ok(Some(MusicItemType::Playlist)) } MusicPageType::None => Ok(None), }, @@ -854,7 +854,7 @@ impl MusicListMapper { pub fn map_response( &mut self, mut res: MapResult>, - ) -> Option { + ) -> Option { let mut etype = None; self.warnings.append(&mut res.warnings); res.c diff --git a/src/client/search.rs b/src/client/search.rs index acee756..4177783 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -5,7 +5,7 @@ use serde::{de::IgnoredAny, Serialize}; use crate::{ deobfuscate::Deobfuscator, error::{Error, ExtractionError}, - model::{Paginator, SearchResult, YouTubeItem}, + model::{paginator::Paginator, SearchResult, YouTubeItem}, param::{search_filter::SearchFilter, Language}, }; @@ -21,6 +21,7 @@ struct QSearch<'a> { } impl RustyPipeQuery { + /// Search YouTube pub async fn search>(&self, query: S) -> Result { let query = query.as_ref(); let context = self.get_context(ClientType::Desktop, true, None).await; @@ -40,6 +41,7 @@ impl RustyPipeQuery { .await } + /// Search YouTube using the given [`SearchFilter`] pub async fn search_filter>( &self, query: S, @@ -63,6 +65,7 @@ impl RustyPipeQuery { .await } + /// Get YouTube search suggestions pub async fn search_suggestion>(&self, query: S) -> Result, Error> { let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t", &[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.as_ref().to_owned())] @@ -114,7 +117,7 @@ impl MapResponse for response::Search { mapper.items, mapper.ctoken, None, - crate::param::ContinuationEndpoint::Search, + crate::model::paginator::ContinuationEndpoint::Search, ), corrected_query: mapper.corrected_query, visitor_data: self.response_context.visitor_data, diff --git a/src/client/trends.rs b/src/client/trends.rs index 8b884ae..6a0fba9 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use crate::{ error::{Error, ExtractionError}, - model::{Paginator, VideoItem}, + model::{paginator::Paginator, VideoItem}, param::Language, serializer::MapResult, util::TryRemove, @@ -11,6 +11,7 @@ use crate::{ use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery}; impl RustyPipeQuery { + /// Get the videos from the YouTube startpage pub async fn startpage(&self) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { @@ -28,6 +29,7 @@ impl RustyPipeQuery { .await } + /// Get the videos from the YouTube trending page pub async fn trending(&self) -> Result, Error> { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QBrowse { @@ -110,7 +112,7 @@ fn map_startpage_videos( mapper.items, mapper.ctoken, visitor_data, - crate::param::ContinuationEndpoint::Browse, + crate::model::paginator::ContinuationEndpoint::Browse, ), warnings: mapper.warnings, } @@ -124,7 +126,7 @@ mod tests { use crate::{ client::{response, MapResponse}, - model::{Paginator, VideoItem}, + model::{paginator::Paginator, VideoItem}, param::Language, serializer::MapResult, }; diff --git a/src/client/url_resolver.rs b/src/client/url_resolver.rs index aa604d3..05377bd 100644 --- a/src/client/url_resolver.rs +++ b/src/client/url_resolver.rs @@ -20,6 +20,39 @@ struct QResolveUrl<'a> { } impl RustyPipeQuery { + /// Resolve the given YouTube URL and return its associated URL target. + /// + /// Note that the hostname of the URL is not checked, so this function also accepts URLs + /// from alternative YouTube frontends like Piped or Invidious. + /// + /// # Examples + /// ``` + /// # use rustypipe::client::RustyPipe; + /// # use rustypipe::model::UrlTarget; + /// # let rp = RustyPipe::new(); + /// # tokio_test::block_on(async { + /// // Channel + /// assert_eq!( + /// rp.query().resolve_url("https://www.youtube.com/LinusTechTips", true).await.unwrap(), + /// UrlTarget::Channel {id: "UCXuqSBlHAE6Xw-yeJA0Tunw".to_owned()} + /// ); + /// // Video + /// assert_eq!( + /// rp.query().resolve_url("https://youtu.be/dQw4w9WgXcQ", true).await.unwrap(), + /// UrlTarget::Video {id: "dQw4w9WgXcQ".to_owned(), start_time: 0} + /// ); + /// // Album + /// // You can choose whether album URLs should be resolved to their album id or returned as playlists + /// assert_eq!( + /// rp.query().resolve_url("https://music.youtube.com/playlist?list=OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE", true).await.unwrap(), + /// UrlTarget::Album {id: "MPREb_GyH43gCvdM5".to_owned()} + /// ); + /// assert_eq!( + /// rp.query().resolve_url("https://music.youtube.com/playlist?list=OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE", false).await.unwrap(), + /// UrlTarget::Playlist {id: "OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE".to_owned()} + /// ); + /// # }); + /// ``` pub async fn resolve_url>( self, url: S, @@ -161,6 +194,29 @@ impl RustyPipeQuery { Ok(target) } + /// Resolve an input string and return a YouTube URL target + /// + /// Accepted input strings include YouTube URLs (see [`RustyPipeQuery::resolve_url`]), + /// Video/Channel/Playlist/Album IDs and channel handles / vanity IDs. + /// + /// # Examples + /// ``` + /// # use rustypipe::client::RustyPipe; + /// # use rustypipe::model::UrlTarget; + /// # let rp = RustyPipe::new(); + /// # tokio_test::block_on(async { + /// // Channel + /// assert_eq!( + /// rp.query().resolve_string("LinusTechTips", true).await.unwrap(), + /// UrlTarget::Channel {id: "UCXuqSBlHAE6Xw-yeJA0Tunw".to_owned()} + /// ); + /// // + /// assert_eq!( + /// rp.query().resolve_string("PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI", true).await.unwrap(), + /// UrlTarget::Playlist {id: "PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI".to_owned()} + /// ); + /// # }); + /// ``` pub async fn resolve_string( self, string: &str, diff --git a/src/client/video_details.rs b/src/client/video_details.rs index 7910556..f1dfc19 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -4,7 +4,7 @@ use serde::Serialize; use crate::{ error::{Error, ExtractionError}, - model::{ChannelTag, Chapter, Comment, Paginator, VideoDetails, VideoItem}, + model::{paginator::Paginator, ChannelTag, Chapter, Comment, VideoDetails, VideoItem}, param::Language, serializer::MapResult, timeago, @@ -28,6 +28,7 @@ struct QVideo<'a> { } impl RustyPipeQuery { + /// Get the metadata for a video pub async fn video_details(&self, video_id: &str) -> Result { let context = self.get_context(ClientType::Desktop, true, None).await; let request_body = QVideo { @@ -47,6 +48,7 @@ impl RustyPipeQuery { .await } + /// Get the comments for a video using the continuation token obtained from `rusty_pipe_query.video_details()` pub async fn video_comments( &self, ctoken: &str, @@ -339,14 +341,14 @@ impl MapResponse for response::VideoDetails { Vec::new(), comment_ctoken, visitor_data.clone(), - crate::param::ContinuationEndpoint::Next, + crate::model::paginator::ContinuationEndpoint::Next, ), latest_comments: Paginator::new_ext( comment_count, Vec::new(), latest_comments_ctoken, visitor_data.clone(), - crate::param::ContinuationEndpoint::Next, + crate::model::paginator::ContinuationEndpoint::Next, ), visitor_data, }, @@ -437,7 +439,7 @@ fn map_recommendations( mapper.items, mapper.ctoken, visitor_data, - crate::param::ContinuationEndpoint::Next, + crate::model::paginator::ContinuationEndpoint::Next, ), warnings: mapper.warnings, } diff --git a/src/error.rs b/src/error.rs index 1ce1641..fa0114e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,3 +1,5 @@ +//! RustyPipe error types + use std::borrow::Cow; /// Custom error type for the RustyPipe library @@ -16,8 +18,10 @@ pub enum Error { /// Error from the HTTP client #[error("http error: {0}")] Http(#[from] reqwest::Error), + /// Erroneous HTTP status code received #[error("http status code: {0}")] HttpStatus(u16), + /// Unspecified error #[error("error: {0}")] Other(Cow<'static, str>), } @@ -33,11 +37,13 @@ pub enum DeobfError { /// Error during JavaScript execution #[error("js execution error: {0}")] JavaScript(#[from] quick_js::ExecutionError), + /// Error during JavaScript parsing #[error("js parsing: {0}")] JsParser(#[from] ress::error::Error), /// Could not extract certain data #[error("could not extract {0}")] Extraction(&'static str), + /// Unspecified error #[error("error: {0}")] Other(&'static str), } @@ -46,22 +52,44 @@ pub enum DeobfError { #[derive(thiserror::Error, Debug)] #[non_exhaustive] pub enum ExtractionError { + /// Video cannot be extracted with RustyPipe + /// + /// Reasons include: + /// - Deletion/Censorship + /// - Private video that requires a Google account + /// - DRM (Movies and TV shows) #[error("Video cant be played because of {0}. Reason (from YT): {1}")] VideoUnavailable(&'static str, String), + /// Video cannot be extracted because it is age restricted. + /// + /// Age restriction may be circumvented with the [`crate::client::ClientType::TvHtml5Embed`] client. #[error("Video is age restricted")] VideoAgeRestricted, + /// Video cannot be extracted because it is not available in your country #[error("Video is not available in your country")] VideoGeoblocked, + /// Video cannot be extracted with the specified client #[error("Video cant be played with this client. Reason (from YT): {0}")] VideoClientUnsupported(String), + /// Content is not available / does not exist #[error("Content is not available. Reason: {0}")] ContentUnavailable(Cow<'static, str>), + /// Error deserializing YouTube's response JSON #[error("deserialization error: {0}")] Deserialization(#[from] serde_json::Error), + /// YouTube returned invalid data #[error("got invalid data from YT: {0}")] InvalidData(Cow<'static, str>), + /// YouTube returned data that does not match the queried ID + /// + /// Specifically YouTube may return this video , + /// which is a 5 minute error message, instead of the requested video when using an outdated + /// Android client. #[error("got wrong result from YT: {0}")] WrongResult(String), + /// Warnings occurred during deserialization/mapping + /// + /// This error is only returned in strict mode. #[error("Warnings during deserialization/mapping")] DeserializationWarnings, } diff --git a/src/lib.rs b/src/lib.rs index f406da5..644fada 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,5 @@ -//! # RustyPipe -//! -//! Client for the public YouTube / YouTube Music API (Innertube), -//! inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). - -#![warn(clippy::todo, clippy::dbg_macro)] +#![doc = include_str!("../README.md")] +#![warn(missing_docs, clippy::todo, clippy::dbg_macro)] #[macro_use] mod macros; diff --git a/src/model/convert.rs b/src/model/convert.rs index b901792..e573911 100644 --- a/src/model/convert.rs +++ b/src/model/convert.rs @@ -3,10 +3,19 @@ use super::{ MusicPlaylistItem, PlaylistItem, TrackItem, VideoItem, YouTubeItem, }; +/// Trait for casting generic YouTube/YouTube music items to a specific kind. +/// +/// Returns [`None`] if the item does not match. pub trait FromYtItem: Sized { + /// Casting from a generic YouTube item to a specific kind + /// + /// Returns [`None`] if the item does not match. fn from_yt_item(_item: YouTubeItem) -> Option { None } + /// Casting from a generic YouTube Music item to a specific kind + /// + /// Returns [`None`] if the item does not match. fn from_ytm_item(_item: MusicItem) -> Option { None } diff --git a/src/model/mod.rs b/src/model/mod.rs index 3d6c80f..ba4c032 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,13 +1,13 @@ -//! YouTube API request and response models +//! YouTube API response models mod convert; mod ordering; -mod paginator; + +pub mod paginator; pub mod richtext; -pub use convert::FromYtItem; -pub use ordering::QualityOrd; -pub use paginator::Paginator; +pub mod traits; + use serde_with::serde_as; use std::{collections::BTreeSet, ops::Range}; @@ -17,7 +17,7 @@ use time::{Date, OffsetDateTime}; use crate::{error::Error, param::Country, serializer::DateYmd, util}; -use self::richtext::RichText; +use self::{paginator::Paginator, richtext::RichText}; /* #COMMON @@ -38,10 +38,36 @@ pub struct Thumbnail { /// Entities extracted from a YouTube URL #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub enum UrlTarget { - Video { id: String, start_time: u32 }, - Channel { id: String }, - Playlist { id: String }, - Album { id: String }, + /// YouTube video + /// + /// Example: + Video { + /// Unique YouTube video ID + id: String, + /// Video start time in seconds + start_time: u32, + }, + /// YouTube channel + /// + /// Example: + Channel { + /// Unique YouTube channel ID + id: String, + }, + /// YouTube playlist + /// + /// Example: + Playlist { + /// Unique YouTube playlist ID + id: String, + }, + /// YouTube Music album + /// + /// Example: + Album { + /// Unique YouTube album ID + id: String, + }, } impl ToString for UrlTarget { @@ -51,10 +77,19 @@ impl ToString for UrlTarget { } impl UrlTarget { + /// Convert the URL target to a YouTube URL + /// + /// Is equivalent to `url_target.to_string()` pub fn to_url(&self) -> String { self.to_url_yt_host("https://www.youtube.com") } + /// Convert the URL target to a YouTube URL with a specified YouTube host. + /// + /// Used to redirect to alternative YouTube frontends like Piped or Invidious. + /// + /// **Note:** Music album URL targets are still converted to `music.youtube.com/browse/*`, + /// since these URLs are not supported by Piped or Invidious. pub fn to_url_yt_host(&self, yt_host: &str) -> String { match self { UrlTarget::Video { id, start_time, .. } => match start_time { @@ -73,6 +108,7 @@ impl UrlTarget { } } + /// Validate the YouTube ID from the URL target pub(crate) fn validate(&self) -> Result<(), Error> { match self { UrlTarget::Video { id, .. } => { @@ -107,11 +143,6 @@ impl UrlTarget { #PLAYER */ -pub trait FileFormat { - /// Get the file extension (".xyz") of the file format - fn extension(&self) -> &str; -} - /// Video player data #[derive(Clone, Debug, Serialize, Deserialize)] #[non_exhaustive] @@ -172,13 +203,17 @@ pub struct VideoStream { pub url: String, /// YouTube stream format identifier pub itag: u32, + /// Stream bitrate (in bits/second) pub bitrate: u32, + /// Average stream bitrate (in bits/second) pub average_bitrate: u32, /// Video file size in bytes pub size: Option, + /// Index range (used for DASH streaming) pub index_range: Option>, + /// Init range (used for DASH streaming) pub init_range: Option>, - // Video duration in milliseconds + /// Video duration in milliseconds pub duration_ms: Option, /// Video width in pixels pub width: u32, @@ -209,13 +244,17 @@ pub struct AudioStream { pub url: String, /// YouTube stream format identifier pub itag: u32, + /// Stream bitrate (in bits/second) pub bitrate: u32, + /// Average stream bitrate (in bits/second) pub average_bitrate: u32, /// Audio file size in bytes pub size: u64, + /// Index range (used for DASH streaming) pub index_range: Option>, + /// Init range (used for DASH streaming) pub init_range: Option>, - // Audio duration in milliseconds + /// Audio duration in milliseconds pub duration_ms: Option, /// MIME file type pub mime: String, @@ -241,94 +280,6 @@ pub struct AudioStream { pub track: Option, } -pub trait YtStream { - fn url(&self) -> &str; - fn itag(&self) -> u32; - fn bitrate(&self) -> u32; - fn averate_bitrate(&self) -> u32; - fn size(&self) -> Option; - fn index_range(&self) -> Option>; - fn init_range(&self) -> Option>; - fn duration_ms(&self) -> Option; - fn mime(&self) -> &str; -} - -impl YtStream for VideoStream { - fn url(&self) -> &str { - &self.url - } - - fn itag(&self) -> u32 { - self.itag - } - - fn bitrate(&self) -> u32 { - self.bitrate - } - - fn averate_bitrate(&self) -> u32 { - self.average_bitrate - } - - fn size(&self) -> Option { - self.size - } - - fn index_range(&self) -> Option> { - self.index_range.clone() - } - - fn init_range(&self) -> Option> { - self.init_range.clone() - } - - fn duration_ms(&self) -> Option { - self.duration_ms - } - - fn mime(&self) -> &str { - &self.mime - } -} - -impl YtStream for AudioStream { - fn url(&self) -> &str { - &self.url - } - - fn itag(&self) -> u32 { - self.itag - } - - fn bitrate(&self) -> u32 { - self.bitrate - } - - fn averate_bitrate(&self) -> u32 { - self.average_bitrate - } - - fn size(&self) -> Option { - Some(self.size) - } - - fn index_range(&self) -> Option> { - self.index_range.clone() - } - - fn init_range(&self) -> Option> { - self.init_range.clone() - } - - fn duration_ms(&self) -> Option { - self.duration_ms - } - - fn mime(&self) -> &str { - &self.mime - } -} - /// Video codec #[derive( Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, @@ -336,6 +287,7 @@ impl YtStream for AudioStream { #[serde(rename_all = "snake_case")] #[non_exhaustive] pub enum VideoCodec { + /// Unknown codec #[default] Unknown, /// MPEG-4 Part 14 @@ -355,6 +307,7 @@ pub enum VideoCodec { #[serde(rename_all = "snake_case")] #[non_exhaustive] pub enum AudioCodec { + /// Unknown codec #[default] Unknown, /// MP4A aka AAC: @@ -396,16 +349,6 @@ pub struct AudioTrack { pub is_default: bool, } -impl FileFormat for VideoFormat { - fn extension(&self) -> &str { - match self { - VideoFormat::ThreeGp => ".3gp", - VideoFormat::Mp4 => ".mp4", - VideoFormat::Webm => ".webm", - } - } -} - /// Audio file type #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "snake_case")] @@ -417,15 +360,6 @@ pub enum AudioFormat { Webm, } -impl FileFormat for AudioFormat { - fn extension(&self) -> &str { - match self { - AudioFormat::M4a => ".m4a", - AudioFormat::Webm => ".webm", - } - } -} - /// YouTube provides subtitles in different formats. /// /// srv1 (XML) is the default format, to request a different format you have @@ -1023,7 +957,9 @@ pub struct ArtistItem { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct ArtistId { + /// Unique YouTube channel ID pub id: Option, + /// Artist name pub name: String, } @@ -1203,10 +1139,12 @@ pub struct MusicSearchResult { pub corrected_query: Option, /// Order of the item sections of the search page, starting with /// the most relevant. - pub order: Vec, + pub order: Vec, } +/// Generic YouTube Music item #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[allow(missing_docs)] pub enum MusicItem { Track(TrackItem), Album(AlbumItem), @@ -1214,18 +1152,21 @@ pub enum MusicItem { Playlist(MusicPlaylistItem), } +/// YouTube Music item type #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum MusicEntityType { +#[allow(missing_docs)] +pub enum MusicItemType { Track, Album, Artist, Playlist, } -/// Filtered YouTube Music search result +/// Filtered YouTube Music search result (one item type) #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct MusicSearchFiltered { + /// Search items pub items: Paginator, /// Corrected search query /// @@ -1239,8 +1180,11 @@ pub struct MusicSearchFiltered { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct TrackDetails { + /// Track metadata pub track: TrackItem, + /// ID to fetch lyrics pub lyrics_id: Option, + /// ID to fetch related tracks pub related_id: Option, } @@ -1254,7 +1198,7 @@ pub struct Lyrics { pub footer: String, } -/// YouTube Music entities related to a track +/// YouTube Music items related to a track #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct MusicRelated { diff --git a/src/model/ordering.rs b/src/model/ordering.rs index 012f35b..b99f7bf 100644 --- a/src/model/ordering.rs +++ b/src/model/ordering.rs @@ -4,7 +4,13 @@ use crate::model::AudioCodec; use super::{AudioStream, VideoStream}; +/// Trait for ordering YouTube video/audio streams by quality +/// +/// analogous to [`std::cmp::Ord`] pub trait QualityOrd { + /// Compare two streams by quality + /// + /// analogous to [`std::cmp::Ord::cmp`] fn quality_cmp(&self, other: &Self) -> Ordering; } diff --git a/src/model/paginator.rs b/src/model/paginator.rs index 366541a..6ca5790 100644 --- a/src/model/paginator.rs +++ b/src/model/paginator.rs @@ -1,9 +1,9 @@ +//! Wrapper model for progressively fetched items + use std::convert::TryInto; use serde::{Deserialize, Serialize}; -use crate::param::ContinuationEndpoint; - /// Wrapper around progressively fetched items /// /// The paginator is a wrapper around a list of items that are fetched @@ -34,7 +34,7 @@ pub struct Paginator { #[serde(skip_serializing_if = "Option::is_none")] pub visitor_data: Option, /// YouTube API endpoint to fetch continuations from - pub endpoint: ContinuationEndpoint, + pub(crate) endpoint: ContinuationEndpoint, } impl Default for Paginator { @@ -49,6 +49,39 @@ impl Default for Paginator { } } +/// YouTube API endpoint to fetch continuations from +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +#[non_exhaustive] +#[allow(missing_docs)] +pub enum ContinuationEndpoint { + Browse, + Search, + Next, + MusicBrowse, + MusicSearch, + MusicNext, +} + +impl ContinuationEndpoint { + pub(crate) fn as_str(self) -> &'static str { + match self { + ContinuationEndpoint::Browse | ContinuationEndpoint::MusicBrowse => "browse", + ContinuationEndpoint::Search | ContinuationEndpoint::MusicSearch => "search", + ContinuationEndpoint::Next | ContinuationEndpoint::MusicNext => "next", + } + } + + pub(crate) fn is_music(self) -> bool { + matches!( + self, + ContinuationEndpoint::MusicBrowse + | ContinuationEndpoint::MusicSearch + | ContinuationEndpoint::MusicNext + ) + } +} + impl Paginator { pub(crate) fn new(count: Option, items: Vec, ctoken: Option) -> Self { Self::new_ext(count, items, ctoken, None, ContinuationEndpoint::Browse) diff --git a/src/model/richtext.rs b/src/model/richtext.rs index 1947e48..836e3f6 100644 --- a/src/model/richtext.rs +++ b/src/model/richtext.rs @@ -6,19 +6,31 @@ use crate::util; use super::UrlTarget; +/// Text content with links #[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub struct RichText(pub Vec); +/// Text component forming a [`RichText`] object #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[non_exhaustive] pub enum TextComponent { /// Plain text Text(String), /// Web link - Web { text: String, url: String }, - /// Link to a YouTube entity - YouTube { text: String, target: UrlTarget }, + Web { + /// Link text + text: String, + /// Link URL + url: String, + }, + /// Link to a YouTube item + YouTube { + /// Link text + text: String, + /// YouTube URL target + target: UrlTarget, + }, } /// Trait for converting rich text to plain text. @@ -46,6 +58,7 @@ pub trait ToHtml { } impl TextComponent { + /// Get the text from the component pub fn get_text(&self) -> &str { match self { TextComponent::Text(text) => text, @@ -54,9 +67,12 @@ impl TextComponent { } } + /// Get the link URL from the component + /// + /// Returns an empty string if the component is not a link. pub fn get_url(&self, yt_host: &str) -> String { match self { - TextComponent::Text(_) => "".to_owned(), + TextComponent::Text(_) => String::new(), TextComponent::Web { url, .. } => url.to_owned(), TextComponent::YouTube { target, .. } => target.to_url_yt_host(yt_host), } diff --git a/src/model/traits.rs b/src/model/traits.rs new file mode 100644 index 0000000..e64c5b2 --- /dev/null +++ b/src/model/traits.rs @@ -0,0 +1,130 @@ +//! Traits for working with response models + +use std::ops::Range; + +pub use super::{convert::FromYtItem, ordering::QualityOrd}; + +use super::{AudioFormat, AudioStream, VideoFormat, VideoStream}; + +/// Trait for YouTube streams (video and audio) +pub trait YtStream { + /// Stream URL + fn url(&self) -> &str; + /// YouTube stream format identifier + fn itag(&self) -> u32; + /// Stream bitrate (in bits/second) + fn bitrate(&self) -> u32; + /// Average stream bitrate (in bits/second) + fn averate_bitrate(&self) -> u32; + /// File size in bytes + fn size(&self) -> Option; + /// Index range (used for DASH streaming) + fn index_range(&self) -> Option>; + /// Init range (used for DASH streaming) + fn init_range(&self) -> Option>; + /// Stream duration in milliseconds + fn duration_ms(&self) -> Option; + /// MIME file type + fn mime(&self) -> &str; +} + +impl YtStream for VideoStream { + fn url(&self) -> &str { + &self.url + } + + fn itag(&self) -> u32 { + self.itag + } + + fn bitrate(&self) -> u32 { + self.bitrate + } + + fn averate_bitrate(&self) -> u32 { + self.average_bitrate + } + + fn size(&self) -> Option { + self.size + } + + fn index_range(&self) -> Option> { + self.index_range.clone() + } + + fn init_range(&self) -> Option> { + self.init_range.clone() + } + + fn duration_ms(&self) -> Option { + self.duration_ms + } + + fn mime(&self) -> &str { + &self.mime + } +} + +impl YtStream for AudioStream { + fn url(&self) -> &str { + &self.url + } + + fn itag(&self) -> u32 { + self.itag + } + + fn bitrate(&self) -> u32 { + self.bitrate + } + + fn averate_bitrate(&self) -> u32 { + self.average_bitrate + } + + fn size(&self) -> Option { + Some(self.size) + } + + fn index_range(&self) -> Option> { + self.index_range.clone() + } + + fn init_range(&self) -> Option> { + self.init_range.clone() + } + + fn duration_ms(&self) -> Option { + self.duration_ms + } + + fn mime(&self) -> &str { + &self.mime + } +} + +/// Trait for file types +pub trait FileFormat { + /// Get the file extension (".xyz") of the file format + fn extension(&self) -> &str; +} + +impl FileFormat for VideoFormat { + fn extension(&self) -> &str { + match self { + VideoFormat::ThreeGp => ".3gp", + VideoFormat::Mp4 => ".mp4", + VideoFormat::Webm => ".webm", + } + } +} + +impl FileFormat for AudioFormat { + fn extension(&self) -> &str { + match self { + AudioFormat::M4a => ".m4a", + AudioFormat::Webm => ".webm", + } + } +} diff --git a/src/param/mod.rs b/src/param/mod.rs index 3f10e91..a3f4708 100644 --- a/src/param/mod.rs +++ b/src/param/mod.rs @@ -1,40 +1,9 @@ +//! Query parameters + mod stream_filter; pub mod locale; pub mod search_filter; pub use locale::{Country, Language}; -use serde::{Deserialize, Serialize}; pub use stream_filter::StreamFilter; - -/// YouTube API endpoint to fetch continuations from -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -#[non_exhaustive] -pub enum ContinuationEndpoint { - Browse, - Search, - Next, - MusicBrowse, - MusicSearch, - MusicNext, -} - -impl ContinuationEndpoint { - pub(crate) fn as_str(self) -> &'static str { - match self { - ContinuationEndpoint::Browse | ContinuationEndpoint::MusicBrowse => "browse", - ContinuationEndpoint::Search | ContinuationEndpoint::MusicSearch => "search", - ContinuationEndpoint::Next | ContinuationEndpoint::MusicNext => "next", - } - } - - pub(crate) fn is_music(self) -> bool { - matches!( - self, - ContinuationEndpoint::MusicBrowse - | ContinuationEndpoint::MusicSearch - | ContinuationEndpoint::MusicNext - ) - } -} diff --git a/src/param/search_filter.rs b/src/param/search_filter.rs index cce0376..becdad2 100644 --- a/src/param/search_filter.rs +++ b/src/param/search_filter.rs @@ -1,13 +1,23 @@ +//! YouTube search filter + use std::collections::BTreeSet; use crate::util::{self, ProtoBuilder}; +/// YouTube search filter +/// +/// Allows you to filter YouTube's search results by +/// item type, features (e.g. HD, 3D, Creative commons), upload date +/// and length. +/// +/// Additionally you can sort the search results by rating, upload date +/// or view count. #[derive(Default, Debug)] pub struct SearchFilter { sort: Option, features: BTreeSet, date: Option, - entity: Option, + item_type: Option, length: Option, verbatim: bool, } @@ -61,9 +71,10 @@ pub enum UploadDate { Year = 5, } -/// YouTube entity type to filter by +/// YouTube item type to filter by #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Entity { +#[allow(missing_docs)] +pub enum ItemType { Video = 1, Channel = 2, Playlist = 3, @@ -81,6 +92,7 @@ pub enum Length { } impl SearchFilter { + /// Get a new [`SearchFilter`] pub fn new() -> Self { Self::default() } @@ -122,14 +134,14 @@ impl SearchFilter { } /// Filter videos by entity type - pub fn entity(mut self, entity: Entity) -> Self { - self.entity = Some(entity); + pub fn item_type(mut self, entity: ItemType) -> Self { + self.item_type = Some(entity); self } /// Filter videos by entity type - pub fn entity_opt(mut self, entity: Option) -> Self { - self.entity = entity; + pub fn item_type_opt(mut self, entity: Option) -> Self { + self.item_type = entity; self } @@ -163,7 +175,7 @@ impl SearchFilter { if let Some(date) = self.date { filters.varint(1, date as u64); } - if let Some(entity) = self.entity { + if let Some(entity) = self.item_type { filters.varint(2, entity as u64); } if let Some(length) = self.length { @@ -200,9 +212,9 @@ mod tests { use super::*; #[rstest] - #[case(SearchFilter::new().entity(Entity::Video), "EgIQAQ%253D%253D")] - #[case(SearchFilter::new().entity(Entity::Channel), "EgIQAg%253D%253D")] - #[case(SearchFilter::new().entity(Entity::Playlist), "EgIQAw%253D%253D")] + #[case(SearchFilter::new().item_type(ItemType::Video), "EgIQAQ%253D%253D")] + #[case(SearchFilter::new().item_type(ItemType::Channel), "EgIQAg%253D%253D")] + #[case(SearchFilter::new().item_type(ItemType::Playlist), "EgIQAw%253D%253D")] #[case(SearchFilter::new().date(UploadDate::Hour), "EgIIAQ%253D%253D")] #[case(SearchFilter::new().date(UploadDate::Day), "EgIIAg%253D%253D")] #[case(SearchFilter::new().date(UploadDate::Week), "EgIIAw%253D%253D")] diff --git a/src/param/stream_filter.rs b/src/param/stream_filter.rs index 5067757..f362316 100644 --- a/src/param/stream_filter.rs +++ b/src/param/stream_filter.rs @@ -3,10 +3,11 @@ use std::cmp::Ordering; use crate::model::{ - AudioCodec, AudioFormat, AudioStream, QualityOrd, VideoCodec, VideoFormat, VideoPlayer, + traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, VideoCodec, VideoFormat, VideoPlayer, VideoStream, }; +/// The StreamFilter is used for selecting audio/video streams from an extracted video #[derive(Debug, Default, Clone)] pub struct StreamFilter<'a> { audio_max_bitrate: Option, @@ -229,6 +230,7 @@ impl<'a> StreamFilter<'a> { } impl VideoPlayer { + /// Select the audio stream which is the best match for the given [`StreamFilter`] pub fn select_audio_stream(&self, filter: &StreamFilter) -> Option<&AudioStream> { let mut fallback: Option<&AudioStream> = None; @@ -282,14 +284,17 @@ impl VideoPlayer { .or(fallback) } + /// Select the video stream which is the best match for the given [`StreamFilter`] pub fn select_video_stream(&self, filter: &StreamFilter) -> Option<&VideoStream> { Self::_select_video_stream(&self.video_streams, filter) } + /// Select the video-only stream which is the best match for the given [`StreamFilter`] pub fn select_video_only_stream(&self, filter: &StreamFilter) -> Option<&VideoStream> { Self::_select_video_stream(&self.video_only_streams, filter) } + /// Select a video and audio stream which is the best match for the given [`StreamFilter`] pub fn select_video_audio_stream( &self, filter: &StreamFilter, diff --git a/src/report.rs b/src/report.rs index 3497cee..07a80f1 100644 --- a/src/report.rs +++ b/src/report.rs @@ -1,4 +1,20 @@ -//! Error reporting +//! # Error reporting +//! +//! Due to the instability of the Innertube API, RustyPipe may not be able to parse +//! every item from every YouTube response. To allow for easy debugging, RustyPipe +//! can create and store error reports. +//! +//! These reports contain information about the RustyPipe client, the performed +//! operation, the request sent to YouTube and the received response data. +//! +//! With the report data the error can be reproduced and RustyPipe can be patched to +//! handle YouTube's changes to the response model. +//! +//! By default, RustyPipe stores the reports as JSON files +//! (e.g `rustypipe_reports/2022-11-05_22-58-59_ERR`). +//! +//! By implementing the [`Reporter`] trait you can handle error reports in other ways +//! (e.g. store them in a database, send them via mail, log to Sentry, etc). use std::{ collections::BTreeMap, @@ -17,11 +33,13 @@ use crate::{deobfuscate::DeobfData, util}; const FILENAME_FORMAT: &[time::format_description::FormatItem] = format_description!("[year]-[month]-[day]_[hour]-[minute]-[second]"); +/// RustyPipe error report #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] pub struct Report { - pub info: Info, - /// Report level + /// Information about the RustyPipe client + pub info: RustyPipeInfo, + /// Severity of the report pub level: Level, /// RustyPipe operation (e.g. `get_player`) pub operation: String, @@ -36,9 +54,10 @@ pub struct Report { pub http_request: HTTPRequest, } +/// Information about the RustyPipe client #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] -pub struct Info { +pub struct RustyPipeInfo { /// Rust package name (`rustypipe`) pub package: String, /// Package version (`0.1.0`) @@ -48,6 +67,7 @@ pub struct Info { pub date: OffsetDateTime, } +/// Reported HTTP request data #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] pub struct HTTPRequest { @@ -65,6 +85,7 @@ pub struct HTTPRequest { pub resp_body: String, } +/// Severity of the report #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Level { /// **Debug**: Operation successful, report generation was forced by setting @@ -76,7 +97,7 @@ pub enum Level { ERR, } -impl Default for Info { +impl Default for RustyPipeInfo { fn default() -> Self { Self { package: "rustypipe".to_owned(), @@ -86,15 +107,20 @@ impl Default for Info { } } +/// Trait used to abstract the report storage behavior, so you can handle RustyPipe's +/// error reports in your preferred way. pub trait Reporter: Sync + Send { + /// Store a RustyPipe error report fn report(&self, report: &Report); } +/// [`Reporter`] implementation that writes reports as JSON files to the given folder pub struct FileReporter { path: PathBuf, } impl FileReporter { + /// Create a new reporter that stores error reports in the given folder pub fn new>(path: P) -> Self { Self { path: path.as_ref().to_path_buf(), diff --git a/src/timeago.rs b/src/timeago.rs index 92634ed..3305925 100644 --- a/src/timeago.rs +++ b/src/timeago.rs @@ -30,7 +30,9 @@ use crate::{ /// Example: "14 hours ago" => `TimeAgo {n: 14, unit: TimeUnit::Hour}` #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct TimeAgo { + /// Number of time units pub n: u8, + /// Time unit pub unit: TimeUnit, } @@ -42,12 +44,20 @@ pub struct TimeAgo { /// - "2 months ago" => `ParsedDate::Relative(TimeAgo {n: 2, unit: TimeUnit::Month})` #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum ParsedDate { + /// Absolute date + /// + /// Example: "Jul 2, 2014" Absolute(Date), + /// Relative date + /// + /// Example: "2 months ago" Relative(TimeAgo), } +/// Parsed time unit #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] #[serde(rename_all = "lowercase")] +#[allow(missing_docs)] pub enum TimeUnit { Second, Minute, diff --git a/tests/youtube.rs b/tests/youtube.rs index 8c4ccb2..79a05a3 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -10,9 +10,11 @@ use time::OffsetDateTime; use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery}; use rustypipe::error::{Error, ExtractionError}; use rustypipe::model::{ - richtext::ToPlaintext, AlbumType, AudioCodec, AudioFormat, Channel, FromYtItem, - MusicEntityType, MusicGenre, Paginator, UrlTarget, Verification, VideoCodec, VideoFormat, - YouTubeItem, YtStream, + paginator::Paginator, + richtext::ToPlaintext, + traits::{FromYtItem, YtStream}, + AlbumType, AudioCodec, AudioFormat, Channel, MusicGenre, MusicItemType, UrlTarget, + Verification, VideoCodec, VideoFormat, YouTubeItem, }; use rustypipe::param::{ search_filter::{self, SearchFilter}, @@ -1109,15 +1111,15 @@ async fn search() { } #[rstest] -#[case::video(search_filter::Entity::Video)] -#[case::channel(search_filter::Entity::Channel)] -#[case::playlist(search_filter::Entity::Playlist)] +#[case::video(search_filter::ItemType::Video)] +#[case::channel(search_filter::ItemType::Channel)] +#[case::playlist(search_filter::ItemType::Playlist)] #[tokio::test] -async fn search_filter_entity(#[case] entity: search_filter::Entity) { +async fn search_filter_item_type(#[case] item_type: search_filter::ItemType) { let rp = RustyPipe::builder().strict().build(); let mut result = rp .query() - .search_filter("with no videos", &SearchFilter::new().entity(entity)) + .search_filter("with no videos", &SearchFilter::new().item_type(item_type)) .await .unwrap(); @@ -1126,13 +1128,13 @@ async fn search_filter_entity(#[case] entity: search_filter::Entity) { result.items.items.iter().for_each(|item| match item { YouTubeItem::Video(_) => { - assert_eq!(entity, search_filter::Entity::Video); + assert_eq!(item_type, search_filter::ItemType::Video); } YouTubeItem::Channel(_) => { - assert_eq!(entity, search_filter::Entity::Channel); + assert_eq!(item_type, search_filter::ItemType::Channel); } YouTubeItem::Playlist(_) => { - assert_eq!(entity, search_filter::Entity::Playlist); + assert_eq!(item_type, search_filter::ItemType::Playlist); } }); } @@ -1249,7 +1251,7 @@ async fn startpage() { // The startpage requires visitor data to fetch continuations assert!(startpage.visitor_data.is_some()); - assert_next(startpage, rp.query(), 20, 2).await; + assert_next(startpage, rp.query(), 15, 2).await; } #[tokio::test] @@ -1508,7 +1510,7 @@ async fn music_search(#[case] typo: bool) { assert!(!res.albums.is_empty(), "no albums"); assert!(!res.artists.is_empty(), "no artists"); assert!(!res.playlists.is_empty(), "no playlists"); - assert_eq!(res.order[0], MusicEntityType::Track); + assert_eq!(res.order[0], MusicItemType::Track); if typo { assert_eq!(res.corrected_query.unwrap(), "black mamba"); @@ -2207,7 +2209,7 @@ async fn ab3_search_channel_handles() { rp.query() .search_filter( "test", - &SearchFilter::new().entity(search_filter::Entity::Channel), + &SearchFilter::new().item_type(search_filter::ItemType::Channel), ) .await .unwrap();