Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor!: refactored response models

Browse source 
doc: documented all public methods
This commit is contained in:
ThetaDev 2022-12-09 01:01:25 +01:00
parent 4c1876cb55
commit f526ab38eb
37 changed files with 600 additions and 255 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -56,5 +56,6 @@ env_logger = "0.10.0"
test-log = "0.2.11"
rstest = "0.16.0"
temp_testdir = "0.2.3"
tokio-test = "0.4.2"
insta = { version = "1.17.1", features = ["ron", "redactions"] }
path_macro = "1.0.0"

4
codegen/src/abtest.rs
View file

@ -4,7 +4,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use num_enum::TryFromPrimitive;
use rustypipe::client::{ClientType, RustyPipe, YTContext};
use rustypipe::model::YouTubeItem;
use rustypipe::param::search_filter::{Entity, SearchFilter};
use rustypipe::param::search_filter::{ItemType, SearchFilter};
use serde::{Deserialize, Serialize};
#[derive(
@ -158,7 +158,7 @@ pub async fn channel_handles_in_search_results(rp: &RustyPipe, visitor_data: &st
let search = rp
.query()
.visitor_data(visitor_data)
.search_filter("rust", &SearchFilter::new().entity(Entity::Channel))
.search_filter("rust", &SearchFilter::new().item_type(ItemType::Channel))
.await
.unwrap();

4
codegen/src/download_testfiles.rs
View file

@ -8,7 +8,7 @@ use std::{
use rustypipe::{
client::{ClientType, RustyPipe},
param::{
search_filter::{self, Entity, SearchFilter},
search_filter::{self, ItemType, SearchFilter},
Country,
},
report::{Report, Reporter},
@ -449,7 +449,7 @@ async fn search_playlists(testfiles: &Path) {
let rp = rp_testfile(&json_path);
rp.query()
.search_filter("pop", &SearchFilter::new().entity(Entity::Playlist))
.search_filter("pop", &SearchFilter::new().item_type(ItemType::Playlist))
.await
.unwrap();
}

2
downloader/src/lib.rs
View file

@ -12,7 +12,7 @@ use rand::Rng;
use regex::Regex;
use reqwest::{header, Client};
use rustypipe::{
model::{AudioCodec, FileFormat, VideoCodec, VideoPlayer},
model::{traits::FileFormat, AudioCodec, VideoCodec, VideoPlayer},
param::StreamFilter,
};
use tokio::{

15
src/cache.rs
View file

@ -7,16 +7,31 @@ use std::{
use log::error;
/// RustyPipe has to cache some information fetched from YouTube: specifically
/// the client versions and the JavaScript code used to deobfuscate the stream URLs.
///
/// This trait is used to abstract the cache storage behavior so you can store
/// cache data in your preferred way (File, SQL, Redis, etc).
///
/// The cache is read when building the [`crate::client::RustyPipe`] client and updated
/// whenever additional data is fetched.
pub trait CacheStorage: Sync + Send {
/// Write the given string to the cache
fn write(&self, data: &str);
/// Read the string from the cache
///
/// Returns [`None`] when the cache is empty or the reading failed.
fn read(&self) -> Option<String>;
}
/// [`CacheStorage`] implementation that writes the cache to a JSON file
/// at the given location.
pub struct FileStorage {
path: PathBuf,
}
impl FileStorage {
/// Create a new JSON-file based cache storage
pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self {
path: path.as_ref().to_path_buf(),

12
src/client/channel.rs
View file

@ -5,7 +5,7 @@ use url::Url;
use crate::{
error::{Error, ExtractionError},
model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem, YouTubeItem},
model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem},
param::Language,
serializer::MapResult,
util,
@ -66,6 +66,7 @@ impl RustyPipeQuery {
.await
}
/// Get the videos from a YouTube channel
pub async fn channel_videos<S: AsRef<str>>(
&self,
channel_id: S,
@ -74,6 +75,7 @@ impl RustyPipeQuery {
.await
}
/// Get the short videos from a YouTube channel
pub async fn channel_shorts<S: AsRef<str>>(
&self,
channel_id: S,
@ -82,6 +84,7 @@ impl RustyPipeQuery {
.await
}
/// Get the livestreams from a YouTube channel
pub async fn channel_livestreams<S: AsRef<str>>(
&self,
channel_id: S,
@ -90,6 +93,7 @@ impl RustyPipeQuery {
.await
}
/// Search the videos of a channel
pub async fn channel_search<S: AsRef<str>, S2: AsRef<str>>(
&self,
channel_id: S,
@ -104,6 +108,7 @@ impl RustyPipeQuery {
.await
}
/// Get the playlists of a channel
pub async fn channel_playlists<S: AsRef<str>>(
&self,
channel_id: S,
@ -127,6 +132,7 @@ impl RustyPipeQuery {
.await
}
/// Get additional metadata from the *About* tab of a channel
pub async fn channel_info<S: AsRef<str>>(
&self,
channel_id: S,
@ -181,7 +187,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
mapper.items,
mapper.ctoken,
self.response_context.visitor_data,
crate::param::ContinuationEndpoint::Browse,
crate::model::paginator::ContinuationEndpoint::Browse,
);
Ok(MapResult {
@ -487,7 +493,7 @@ mod tests {
use crate::{
client::{response, MapResponse},
model::{Channel, ChannelInfo, Paginator, PlaylistItem, VideoItem},
model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem},
param::Language,
serializer::MapResult,
};

6
src/client/channel_rss.rs
View file

@ -9,6 +9,12 @@ use crate::{
use super::{response, RustyPipeQuery};
impl RustyPipeQuery {
/// Get the 15 latest videos from the channel's RSS feed
///
/// Example: <https://www.youtube.com/feeds/videos.xml?channel_id=UC2DjFE7Xf11URZqWBigcVOQ>
///
/// Fetching RSS feeds is a lot faster than querying the InnerTube API, so this method is great
/// for checking a lot of channels or implementing a subscription feed.
pub async fn channel_rss<S: AsRef<str>>(&self, channel_id: S) -> Result<ChannelRss, Error> {
let url = format!(
"https://www.youtube.com/feeds/videos.xml?channel_id={}",

28
src/client/mod.rs
View file

@ -48,20 +48,26 @@ use crate::{
///
/// There are multiple clients for accessing the YouTube API which have
/// slightly different features
///
/// - **Desktop**: used by youtube.com
/// - **DesktopMusic**: used by music.youtube.com, can access special music data,
/// cannot access non-music content
/// - **TvHtml5Embed**: used by Smart TVs, can access age-restricted videos
/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams
/// - **Ios**: used by the iOS app, no obfuscated URLs
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum ClientType {
/// Client used by youtube.com
Desktop,
/// Client used by music.youtube.com
///
/// can access YTM-specific data, cannot access non-music content
DesktopMusic,
/// used by Smart TVs
///
/// can access age-restricted videos, cannot access non-embeddable videos
TvHtml5Embed,
/// used by the Android app
///
/// no obfuscated stream URLs, includes lower resolution audio streams
Android,
/// used by the iOS app
///
/// no obfuscated stream URLs
Ios,
}
@ -74,6 +80,7 @@ impl ClientType {
}
}
/// YouTube context request parameter
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct YTContext<'a> {
@ -204,6 +211,7 @@ struct RustyPipeOpts {
visitor_data: Option<String>,
}
/// Builder to construct a new RustyPipe client
pub struct RustyPipeBuilder {
storage: Option<Box<dyn CacheStorage>>,
reporter: Option<Box<dyn Reporter>>,
@ -212,6 +220,10 @@ pub struct RustyPipeBuilder {
default_opts: RustyPipeOpts,
}
/// RustyPipe query object
///
/// Contains a reference to the RustyPipe client as well as query-specific
/// options (e.g. language preference).
#[derive(Clone)]
pub struct RustyPipeQuery {
client: RustyPipe,
@ -257,7 +269,7 @@ enum CacheEntry<T> {
}
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ClientData {
struct ClientData {
pub version: String,
}

3
src/client/music_artist.rs
View file

@ -26,6 +26,9 @@ struct QBrowseParams<'a> {
}
impl RustyPipeQuery {
/// Get a YouTube Music artist page
///
/// Set `all_albums` to [`true`] if you want to fetch the albums behind the *More* buttons, too.
pub async fn music_artist<S: AsRef<str>>(
&self,
artist_id: S,

1
src/client/music_charts.rs
View file

@ -31,6 +31,7 @@ struct FormData {
}
impl RustyPipeQuery {
/// Get the YouTube Music charts for a given country
pub async fn music_charts(&self, country: Option<Country>) -> Result<MusicCharts, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QCharts {

16
src/client/music_details.rs
View file

@ -4,7 +4,7 @@ use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{ArtistId, Lyrics, MusicRelated, Paginator, TrackDetails, TrackItem},
model::{paginator::Paginator, ArtistId, Lyrics, MusicRelated, TrackDetails, TrackItem},
param::Language,
serializer::MapResult,
};
@ -37,6 +37,7 @@ struct QRadio<'a> {
}
impl RustyPipeQuery {
/// Get the metadata of a YouTube music track
pub async fn music_details<S: AsRef<str>>(&self, video_id: S) -> Result<TrackDetails, Error> {
let video_id = video_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
@ -58,6 +59,9 @@ impl RustyPipeQuery {
.await
}
/// Get the lyrics of a YouTube music track
///
/// The `lyrics_id` has to be obtained using [`RustyPipeQuery::music_details`].
pub async fn music_lyrics<S: AsRef<str>>(&self, lyrics_id: S) -> Result<Lyrics, Error> {
let lyrics_id = lyrics_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
@ -76,6 +80,9 @@ impl RustyPipeQuery {
.await
}
/// Get related items (tracks, playlists, artists) to a YouTube Music track
///
/// The `related_id` has to be obtained using [`RustyPipeQuery::music_details`].
pub async fn music_related<S: AsRef<str>>(&self, related_id: S) -> Result<MusicRelated, Error> {
let related_id = related_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
@ -94,6 +101,9 @@ impl RustyPipeQuery {
.await
}
/// Get a YouTube Music radio (a dynamically generated playlist)
///
/// The `radio_id` can be obtained using [`RustyPipeQuery::music_artist`] to get an artist's radio.
pub async fn music_radio<S: AsRef<str>>(
&self,
radio_id: S,
@ -122,6 +132,7 @@ impl RustyPipeQuery {
.await
}
/// Get a YouTube Music radio (a dynamically generated playlist) for a track
pub async fn music_radio_track<S: AsRef<str>>(
&self,
video_id: S,
@ -130,6 +141,7 @@ impl RustyPipeQuery {
.await
}
/// Get a YouTube Music radio (a dynamically generated playlist) for a playlist
pub async fn music_radio_playlist<S: AsRef<str>>(
&self,
playlist_id: S,
@ -263,7 +275,7 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
tracks,
ctoken,
None,
crate::param::ContinuationEndpoint::MusicNext,
crate::model::paginator::ContinuationEndpoint::MusicNext,
),
warnings: content.contents.warnings,
})

2
src/client/music_genres.rs
View file

@ -22,6 +22,7 @@ struct QGenre<'a> {
}
impl RustyPipeQuery {
/// Get a list of moods and genres from YouTube Music
pub async fn music_genres(&self) -> Result<Vec<MusicGenreItem>, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
@ -39,6 +40,7 @@ impl RustyPipeQuery {
.await
}
/// Get the playlists from a YouTube Music genre
pub async fn music_genre<S: AsRef<str>>(&self, genre_id: S) -> Result<MusicGenre, Error> {
let genre_id = genre_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;

4
src/client/music_new.rs
View file

@ -3,12 +3,13 @@ use std::borrow::Cow;
use crate::{
client::response::music_item::MusicListMapper,
error::{Error, ExtractionError},
model::{AlbumItem, FromYtItem, TrackItem},
model::{traits::FromYtItem, AlbumItem, TrackItem},
};
use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery};
impl RustyPipeQuery {
/// Get the new albums that were released on YouTube Music
pub async fn music_new_albums(&self) -> Result<Vec<AlbumItem>, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {
@ -26,6 +27,7 @@ impl RustyPipeQuery {
.await
}
/// Get the new music videos that were released on YouTube Music
pub async fn music_new_videos(&self) -> Result<Vec<TrackItem>, Error> {
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
let request_body = QBrowse {

8
src/client/music_playlist.rs
View file

@ -2,7 +2,7 @@ use std::borrow::Cow;
use crate::{
error::{Error, ExtractionError},
model::{AlbumId, ChannelId, MusicAlbum, MusicPlaylist, Paginator, TrackItem},
model::{paginator::Paginator, AlbumId, ChannelId, MusicAlbum, MusicPlaylist, TrackItem},
serializer::MapResult,
util::{self, TryRemove},
};
@ -16,6 +16,7 @@ use super::{
};
impl RustyPipeQuery {
/// Get a playlist from YouTube Music
pub async fn music_playlist<S: AsRef<str>>(
&self,
playlist_id: S,
@ -37,6 +38,7 @@ impl RustyPipeQuery {
.await
}
/// Get an album from YouTube Music
pub async fn music_album<S: AsRef<str>>(&self, album_id: S) -> Result<MusicAlbum, Error> {
let album_id = album_id.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
@ -221,14 +223,14 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
map_res.c,
ctoken,
None,
crate::param::ContinuationEndpoint::MusicBrowse,
crate::model::paginator::ContinuationEndpoint::MusicBrowse,
),
related_playlists: Paginator::new_ext(
None,
Vec::new(),
related_ctoken,
None,
crate::param::ContinuationEndpoint::MusicBrowse,
crate::model::paginator::ContinuationEndpoint::MusicBrowse,
),
},
warnings: map_res.warnings,

15
src/client/music_search.rs
View file

@ -6,8 +6,8 @@ use crate::{
client::response::music_item::MusicListMapper,
error::{Error, ExtractionError},
model::{
AlbumItem, ArtistItem, FromYtItem, MusicPlaylistItem, MusicSearchFiltered,
MusicSearchResult, Paginator, TrackItem,
paginator::Paginator, traits::FromYtItem, AlbumItem, ArtistItem, MusicPlaylistItem,
MusicSearchFiltered, MusicSearchResult, TrackItem,
},
serializer::MapResult,
util::TryRemove,
@ -50,6 +50,7 @@ enum Params {
}
impl RustyPipeQuery {
/// Search YouTube Music. Returns items from any type.
pub async fn music_search<S: AsRef<str>>(&self, query: S) -> Result<MusicSearchResult, Error> {
let query = query.as_ref();
let context = self.get_context(ClientType::DesktopMusic, true, None).await;
@ -69,6 +70,7 @@ impl RustyPipeQuery {
.await
}
/// Search YouTube Music tracks
pub async fn music_search_tracks<S: AsRef<str>>(
&self,
query: S,
@ -76,6 +78,7 @@ impl RustyPipeQuery {
self._music_search_tracks(query, Params::Tracks).await
}
/// Search YouTube Music videos
pub async fn music_search_videos<S: AsRef<str>>(
&self,
query: S,
@ -106,6 +109,7 @@ impl RustyPipeQuery {
.await
}
/// Search YouTube Music albums
pub async fn music_search_albums<S: AsRef<str>>(
&self,
query: S,
@ -128,6 +132,7 @@ impl RustyPipeQuery {
.await
}
/// Search YouTube Music artists
pub async fn music_search_artists(
&self,
query: &str,
@ -149,6 +154,7 @@ impl RustyPipeQuery {
.await
}
/// Search YouTube Music playlists
pub async fn music_search_playlists<S: AsRef<str>>(
&self,
query: S,
@ -156,6 +162,8 @@ impl RustyPipeQuery {
self._music_search_playlists(query, Params::Playlists).await
}
/// Search YouTube Music playlists that were created by users
/// (`community=true`) or by YouTube Music (`community=false`)
pub async fn music_search_playlists_filter<S: AsRef<str>>(
&self,
query: S,
@ -194,6 +202,7 @@ impl RustyPipeQuery {
.await
}
/// Get YouTube Music search suggestions
pub async fn music_search_suggestion<S: AsRef<str>>(
&self,
query: S,
@ -316,7 +325,7 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
map_res.c,
ctoken,
None,
crate::param::ContinuationEndpoint::MusicSearch,
crate::model::paginator::ContinuationEndpoint::MusicSearch,
),
corrected_query,
},

25
src/client/pagination.rs
View file

@ -1,8 +1,11 @@
use std::borrow::Cow;
use crate::error::{Error, ExtractionError};
use crate::model::{Comment, FromYtItem, MusicItem, Paginator, PlaylistVideo, YouTubeItem};
use crate::param::ContinuationEndpoint;
use crate::model::{
paginator::{ContinuationEndpoint, Paginator},
traits::FromYtItem,
Comment, MusicItem, PlaylistVideo, YouTubeItem,
};
use crate::serializer::MapResult;
use crate::util::TryRemove;
@ -10,6 +13,7 @@ use super::response::music_item::{map_queue_item, MusicListMapper, PlaylistPanel
use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery};
impl RustyPipeQuery {
/// Get more YouTube items from the given continuation token and endpoint
pub async fn continuation<T: FromYtItem, S: AsRef<str>>(
&self,
ctoken: S,
@ -174,6 +178,7 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
}
impl<T: FromYtItem> Paginator<T> {
/// Get the next page from the paginator (or `None` if the paginator is exhausted)
pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(
@ -186,6 +191,9 @@ impl<T: FromYtItem> Paginator<T> {
})
}
/// Extend the items of the paginator by the next page
///
/// Returns false if the paginator is exhausted.
pub async fn extend<Q: AsRef<RustyPipeQuery>>(&mut self, query: Q) -> Result<bool, Error> {
match self.next(query).await {
Ok(Some(paginator)) => {
@ -199,6 +207,8 @@ impl<T: FromYtItem> Paginator<T> {
}
}
/// Extend the items of the paginator by the given amount of pages
/// or until the paginator is exhausted.
pub async fn extend_pages<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
@ -215,6 +225,8 @@ impl<T: FromYtItem> Paginator<T> {
Ok(())
}
/// Extend the items of the paginator until the given amount of items
/// is reached or the paginator is exhausted.
pub async fn extend_limit<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
@ -233,6 +245,7 @@ impl<T: FromYtItem> Paginator<T> {
}
impl Paginator<Comment> {
/// Get the next page from the paginator (or `None` if the paginator is exhausted)
pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(
@ -247,6 +260,7 @@ impl Paginator<Comment> {
}
impl Paginator<PlaylistVideo> {
/// Get the next page from the paginator (or `None` if the paginator is exhausted)
pub async fn next<Q: AsRef<RustyPipeQuery>>(&self, query: Q) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(query.as_ref().playlist_continuation(ctoken).await?),
@ -258,6 +272,9 @@ impl Paginator<PlaylistVideo> {
macro_rules! paginator {
($entity_type:ty) => {
impl Paginator<$entity_type> {
/// Extend the items of the paginator by the next page
///
/// Returns false if the paginator is exhausted.
pub async fn extend<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
@ -274,6 +291,8 @@ macro_rules! paginator {
}
}
/// Extend the items of the paginator by the given amount of pages
/// or until the paginator is exhausted.
pub async fn extend_pages<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,
@ -290,6 +309,8 @@ macro_rules! paginator {
Ok(())
}
/// Extend the items of the paginator until the given amount of items
/// is reached or the paginator is exhausted.
pub async fn extend_limit<Q: AsRef<RustyPipeQuery>>(
&mut self,
query: Q,

4
src/client/player.rs
View file

@ -12,7 +12,7 @@ use crate::{
deobfuscate::Deobfuscator,
error::{DeobfError, Error, ExtractionError},
model::{
AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, QualityOrd, Subtitle,
traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, AudioTrack, ChannelId, Subtitle,
VideoCodec, VideoFormat, VideoPlayer, VideoPlayerDetails, VideoStream,
},
param::Language,
@ -58,6 +58,7 @@ struct QContentPlaybackContext {
}
impl RustyPipeQuery {
/// Get YouTube player data (video/audio streams + basic metadata)
pub async fn player<S: AsRef<str>>(&self, video_id: S) -> Result<VideoPlayer, Error> {
let video_id = video_id.as_ref();
let q1 = self.clone();
@ -86,6 +87,7 @@ impl RustyPipeQuery {
}
}
/// Get YouTube player data (video/audio streams + basic metadata) using the specified client
pub async fn player_from_client<S: AsRef<str>>(
&self,
video_id: S,

4
src/client/playlist.rs
View file

@ -5,7 +5,7 @@ use time::OffsetDateTime;
use crate::{
deobfuscate::Deobfuscator,
error::{Error, ExtractionError},
model::{ChannelId, Paginator, Playlist, PlaylistVideo},
model::{paginator::Paginator, ChannelId, Playlist, PlaylistVideo},
param::Language,
timeago,
util::{self, TryRemove},
@ -14,6 +14,7 @@ use crate::{
use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery};
impl RustyPipeQuery {
/// Get a YouTube playlist
pub async fn playlist<S: AsRef<str>>(&self, playlist_id: S) -> Result<Playlist, Error> {
let playlist_id = playlist_id.as_ref();
let context = self.get_context(ClientType::Desktop, true, None).await;
@ -32,6 +33,7 @@ impl RustyPipeQuery {
.await
}
/// Get more playlist items using the given continuation token
pub async fn playlist_continuation<S: AsRef<str>>(
&self,
ctoken: S,

24
src/client/response/music_item.rs
View file

@ -3,8 +3,8 @@ use serde_with::{rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkip
use crate::{
model::{
self, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId, FromYtItem,
MusicEntityType, MusicItem, MusicPlaylistItem, TrackItem,
self, traits::FromYtItem, AlbumId, AlbumItem, AlbumType, ArtistId, ArtistItem, ChannelId,
MusicItem, MusicItemType, MusicPlaylistItem, TrackItem,
},
param::Language,
serializer::{
@ -465,7 +465,7 @@ impl MusicListMapper {
}
}
fn map_item(&mut self, item: MusicResponseItem) -> Result<Option<MusicEntityType>, String> {
fn map_item(&mut self, item: MusicResponseItem) -> Result<Option<MusicItemType>, String> {
match item {
// List item
MusicResponseItem::MusicResponsiveListItemRenderer(item) => {
@ -636,7 +636,7 @@ impl MusicListMapper {
is_video,
track_nr,
}));
Ok(Some(MusicEntityType::Track))
Ok(Some(MusicItemType::Track))
}
// Artist / Album / Playlist
Some((page_type, id)) => {
@ -666,7 +666,7 @@ impl MusicListMapper {
avatar: item.thumbnail.into(),
subscriber_count,
}));
Ok(Some(MusicEntityType::Artist))
Ok(Some(MusicItemType::Artist))
}
MusicPageType::Album => {
let album_type = subtitle_p1
@ -690,7 +690,7 @@ impl MusicListMapper {
year,
by_va,
}));
Ok(Some(MusicEntityType::Album))
Ok(Some(MusicItemType::Album))
}
MusicPageType::Playlist => {
// Part 1 may be the "Playlist" label
@ -717,7 +717,7 @@ impl MusicListMapper {
track_count,
from_ytm,
}));
Ok(Some(MusicEntityType::Playlist))
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => {
// There may be broken YT channels from the artist search. They can be skipped.
@ -762,7 +762,7 @@ impl MusicListMapper {
is_video,
track_nr: None,
}));
Ok(Some(MusicEntityType::Track))
Ok(Some(MusicItemType::Track))
}
MusicPageType::Artist => {
let subscriber_count = subtitle_p1
@ -774,7 +774,7 @@ impl MusicListMapper {
avatar: item.thumbnail_renderer.into(),
subscriber_count,
}));
Ok(Some(MusicEntityType::Artist))
Ok(Some(MusicItemType::Artist))
}
MusicPageType::Album => {
let mut year = None;
@ -818,7 +818,7 @@ impl MusicListMapper {
year,
by_va,
}));
Ok(Some(MusicEntityType::Album))
Ok(Some(MusicItemType::Album))
}
MusicPageType::Playlist => {
// When the playlist subtitle has only 1 part, it is a playlist from YT Music
@ -841,7 +841,7 @@ impl MusicListMapper {
track_count,
from_ytm,
}));
Ok(Some(MusicEntityType::Playlist))
Ok(Some(MusicItemType::Playlist))
}
MusicPageType::None => Ok(None),
},
@ -854,7 +854,7 @@ impl MusicListMapper {
pub fn map_response(
&mut self,
mut res: MapResult<Vec<MusicResponseItem>>,
) -> Option<MusicEntityType> {
) -> Option<MusicItemType> {
let mut etype = None;
self.warnings.append(&mut res.warnings);
res.c

7
src/client/search.rs
View file

@ -5,7 +5,7 @@ use serde::{de::IgnoredAny, Serialize};
use crate::{
deobfuscate::Deobfuscator,
error::{Error, ExtractionError},
model::{Paginator, SearchResult, YouTubeItem},
model::{paginator::Paginator, SearchResult, YouTubeItem},
param::{search_filter::SearchFilter, Language},
};
@ -21,6 +21,7 @@ struct QSearch<'a> {
}
impl RustyPipeQuery {
/// Search YouTube
pub async fn search<S: AsRef<str>>(&self, query: S) -> Result<SearchResult, Error> {
let query = query.as_ref();
let context = self.get_context(ClientType::Desktop, true, None).await;
@ -40,6 +41,7 @@ impl RustyPipeQuery {
.await
}
/// Search YouTube using the given [`SearchFilter`]
pub async fn search_filter<S: AsRef<str>>(
&self,
query: S,
@ -63,6 +65,7 @@ impl RustyPipeQuery {
.await
}
/// Get YouTube search suggestions
pub async fn search_suggestion<S: AsRef<str>>(&self, query: S) -> Result<Vec<String>, Error> {
let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t",
&[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.as_ref().to_owned())]
@ -114,7 +117,7 @@ impl MapResponse<SearchResult> for response::Search {
mapper.items,
mapper.ctoken,
None,
crate::param::ContinuationEndpoint::Search,
crate::model::paginator::ContinuationEndpoint::Search,
),
corrected_query: mapper.corrected_query,
visitor_data: self.response_context.visitor_data,

8
src/client/trends.rs
View file

@ -2,7 +2,7 @@ use std::borrow::Cow;
use crate::{
error::{Error, ExtractionError},
model::{Paginator, VideoItem},
model::{paginator::Paginator, VideoItem},
param::Language,
serializer::MapResult,
util::TryRemove,
@ -11,6 +11,7 @@ use crate::{
use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery};
impl RustyPipeQuery {
/// Get the videos from the YouTube startpage
pub async fn startpage(&self) -> Result<Paginator<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QBrowse {
@ -28,6 +29,7 @@ impl RustyPipeQuery {
.await
}
/// Get the videos from the YouTube trending page
pub async fn trending(&self) -> Result<Vec<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QBrowse {
@ -110,7 +112,7 @@ fn map_startpage_videos(
mapper.items,
mapper.ctoken,
visitor_data,
crate::param::ContinuationEndpoint::Browse,
crate::model::paginator::ContinuationEndpoint::Browse,
),
warnings: mapper.warnings,
}
@ -124,7 +126,7 @@ mod tests {
use crate::{
client::{response, MapResponse},
model::{Paginator, VideoItem},
model::{paginator::Paginator, VideoItem},
param::Language,
serializer::MapResult,
};

56
src/client/url_resolver.rs
View file

@ -20,6 +20,39 @@ struct QResolveUrl<'a> {
}
impl RustyPipeQuery {
/// Resolve the given YouTube URL and return its associated URL target.
///
/// Note that the hostname of the URL is not checked, so this function also accepts URLs
/// from alternative YouTube frontends like Piped or Invidious.
///
/// # Examples
/// ```
/// # use rustypipe::client::RustyPipe;
/// # use rustypipe::model::UrlTarget;
/// # let rp = RustyPipe::new();
/// # tokio_test::block_on(async {
/// // Channel
/// assert_eq!(
/// rp.query().resolve_url("https://www.youtube.com/LinusTechTips", true).await.unwrap(),
/// UrlTarget::Channel {id: "UCXuqSBlHAE6Xw-yeJA0Tunw".to_owned()}
/// );
/// // Video
/// assert_eq!(
/// rp.query().resolve_url("https://youtu.be/dQw4w9WgXcQ", true).await.unwrap(),
/// UrlTarget::Video {id: "dQw4w9WgXcQ".to_owned(), start_time: 0}
/// );
/// // Album
/// // You can choose whether album URLs should be resolved to their album id or returned as playlists
/// assert_eq!(
/// rp.query().resolve_url("https://music.youtube.com/playlist?list=OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE", true).await.unwrap(),
/// UrlTarget::Album {id: "MPREb_GyH43gCvdM5".to_owned()}
/// );
/// assert_eq!(
/// rp.query().resolve_url("https://music.youtube.com/playlist?list=OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE", false).await.unwrap(),
/// UrlTarget::Playlist {id: "OLAK5uy_k0yFrZlFRgCf3rLPza-lkRmCrtLPbK9pE".to_owned()}
/// );
/// # });
/// ```
pub async fn resolve_url<S: AsRef<str>>(
self,
url: S,
@ -161,6 +194,29 @@ impl RustyPipeQuery {
Ok(target)
}
/// Resolve an input string and return a YouTube URL target
///
/// Accepted input strings include YouTube URLs (see [`RustyPipeQuery::resolve_url`]),
/// Video/Channel/Playlist/Album IDs and channel handles / vanity IDs.
///
/// # Examples
/// ```
/// # use rustypipe::client::RustyPipe;
/// # use rustypipe::model::UrlTarget;
/// # let rp = RustyPipe::new();
/// # tokio_test::block_on(async {
/// // Channel
/// assert_eq!(
/// rp.query().resolve_string("LinusTechTips", true).await.unwrap(),
/// UrlTarget::Channel {id: "UCXuqSBlHAE6Xw-yeJA0Tunw".to_owned()}
/// );
/// //
/// assert_eq!(
/// rp.query().resolve_string("PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI", true).await.unwrap(),
/// UrlTarget::Playlist {id: "PL4lEESSgxM_5O81EvKCmBIm_JT5Q7JeaI".to_owned()}
/// );
/// # });
/// ```
pub async fn resolve_string(
self,
string: &str,

10
src/client/video_details.rs
View file

@ -4,7 +4,7 @@ use serde::Serialize;
use crate::{
error::{Error, ExtractionError},
model::{ChannelTag, Chapter, Comment, Paginator, VideoDetails, VideoItem},
model::{paginator::Paginator, ChannelTag, Chapter, Comment, VideoDetails, VideoItem},
param::Language,
serializer::MapResult,
timeago,
@ -28,6 +28,7 @@ struct QVideo<'a> {
}
impl RustyPipeQuery {
/// Get the metadata for a video
pub async fn video_details(&self, video_id: &str) -> Result<VideoDetails, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QVideo {
@ -47,6 +48,7 @@ impl RustyPipeQuery {
.await
}
/// Get the comments for a video using the continuation token obtained from `rusty_pipe_query.video_details()`
pub async fn video_comments(
&self,
ctoken: &str,
@ -339,14 +341,14 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
Vec::new(),
comment_ctoken,
visitor_data.clone(),
crate::param::ContinuationEndpoint::Next,
crate::model::paginator::ContinuationEndpoint::Next,
),
latest_comments: Paginator::new_ext(
comment_count,
Vec::new(),
latest_comments_ctoken,
visitor_data.clone(),
crate::param::ContinuationEndpoint::Next,
crate::model::paginator::ContinuationEndpoint::Next,
),
visitor_data,
},
@ -437,7 +439,7 @@ fn map_recommendations(
mapper.items,
mapper.ctoken,
visitor_data,
crate::param::ContinuationEndpoint::Next,
crate::model::paginator::ContinuationEndpoint::Next,
),
warnings: mapper.warnings,
}

28
src/error.rs
View file

@ -1,3 +1,5 @@
//! RustyPipe error types
use std::borrow::Cow;
/// Custom error type for the RustyPipe library
@ -16,8 +18,10 @@ pub enum Error {
/// Error from the HTTP client
#[error("http error: {0}")]
Http(#[from] reqwest::Error),
/// Erroneous HTTP status code received
#[error("http status code: {0}")]
HttpStatus(u16),
/// Unspecified error
#[error("error: {0}")]
Other(Cow<'static, str>),
}
@ -33,11 +37,13 @@ pub enum DeobfError {
/// Error during JavaScript execution
#[error("js execution error: {0}")]
JavaScript(#[from] quick_js::ExecutionError),
/// Error during JavaScript parsing
#[error("js parsing: {0}")]
JsParser(#[from] ress::error::Error),
/// Could not extract certain data
#[error("could not extract {0}")]
Extraction(&'static str),
/// Unspecified error
#[error("error: {0}")]
Other(&'static str),
}
@ -46,22 +52,44 @@ pub enum DeobfError {
#[derive(thiserror::Error, Debug)]
#[non_exhaustive]
pub enum ExtractionError {
/// Video cannot be extracted with RustyPipe
///
/// Reasons include:
/// - Deletion/Censorship
/// - Private video that requires a Google account
/// - DRM (Movies and TV shows)
#[error("Video cant be played because of {0}. Reason (from YT): {1}")]
VideoUnavailable(&'static str, String),
/// Video cannot be extracted because it is age restricted.
///
/// Age restriction may be circumvented with the [`crate::client::ClientType::TvHtml5Embed`] client.
#[error("Video is age restricted")]
VideoAgeRestricted,
/// Video cannot be extracted because it is not available in your country
#[error("Video is not available in your country")]
VideoGeoblocked,
/// Video cannot be extracted with the specified client
#[error("Video cant be played with this client. Reason (from YT): {0}")]
VideoClientUnsupported(String),
/// Content is not available / does not exist
#[error("Content is not available. Reason: {0}")]
ContentUnavailable(Cow<'static, str>),
/// Error deserializing YouTube's response JSON
#[error("deserialization error: {0}")]
Deserialization(#[from] serde_json::Error),
/// YouTube returned invalid data
#[error("got invalid data from YT: {0}")]
InvalidData(Cow<'static, str>),
/// YouTube returned data that does not match the queried ID
///
/// Specifically YouTube may return this video <https://www.youtube.com/watch?v=aQvGIIdgFDM>,
/// which is a 5 minute error message, instead of the requested video when using an outdated
/// Android client.
#[error("got wrong result from YT: {0}")]
WrongResult(String),
/// Warnings occurred during deserialization/mapping
///
/// This error is only returned in strict mode.
#[error("Warnings during deserialization/mapping")]
DeserializationWarnings,
}

8
src/lib.rs
View file

@ -1,9 +1,5 @@
//! # RustyPipe
//!
//! Client for the public YouTube / YouTube Music API (Innertube),
//! inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor).
#![warn(clippy::todo, clippy::dbg_macro)]
#![doc = include_str!("../README.md")]
#![warn(missing_docs, clippy::todo, clippy::dbg_macro)]
#[macro_use]
mod macros;

9
src/model/convert.rs
View file

@ -3,10 +3,19 @@ use super::{
MusicPlaylistItem, PlaylistItem, TrackItem, VideoItem, YouTubeItem,
};
/// Trait for casting generic YouTube/YouTube music items to a specific kind.
///
/// Returns [`None`] if the item does not match.
pub trait FromYtItem: Sized {
/// Casting from a generic YouTube item to a specific kind
///
/// Returns [`None`] if the item does not match.
fn from_yt_item(_item: YouTubeItem) -> Option<Self> {
None
}
/// Casting from a generic YouTube Music item to a specific kind
///
/// Returns [`None`] if the item does not match.
fn from_ytm_item(_item: MusicItem) -> Option<Self> {
None
}

200
src/model/mod.rs
View file

@ -1,13 +1,13 @@
//! YouTube API request and response models
//! YouTube API response models
mod convert;
mod ordering;
mod paginator;
pub mod paginator;
pub mod richtext;
pub use convert::FromYtItem;
pub use ordering::QualityOrd;
pub use paginator::Paginator;
pub mod traits;
use serde_with::serde_as;
use std::{collections::BTreeSet, ops::Range};
@ -17,7 +17,7 @@ use time::{Date, OffsetDateTime};
use crate::{error::Error, param::Country, serializer::DateYmd, util};
use self::richtext::RichText;
use self::{paginator::Paginator, richtext::RichText};
/*
#COMMON
@ -38,10 +38,36 @@ pub struct Thumbnail {
/// Entities extracted from a YouTube URL
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub enum UrlTarget {
Video { id: String, start_time: u32 },
Channel { id: String },
Playlist { id: String },
Album { id: String },
/// YouTube video
///
/// Example: <youtube.com/watch?v=ZeerrnuLi5E>
Video {
/// Unique YouTube video ID
id: String,
/// Video start time in seconds
start_time: u32,
},
/// YouTube channel
///
/// Example: <https://www.youtube.com/channel/UC2DjFE7Xf11URZqWBigcVOQ>
Channel {
/// Unique YouTube channel ID
id: String,
},
/// YouTube playlist
///
/// Example: <https://www.youtube.com/playlist?list=PLKUA473MWUv2jmkqIxzQR3YL4kuPArj4G>
Playlist {
/// Unique YouTube playlist ID
id: String,
},
/// YouTube Music album
///
/// Example: <https://music.youtube.com/browse/MPREb_nlBWQROfvjo>
Album {
/// Unique YouTube album ID
id: String,
},
}
impl ToString for UrlTarget {
@ -51,10 +77,19 @@ impl ToString for UrlTarget {
}
impl UrlTarget {
/// Convert the URL target to a YouTube URL
///
/// Is equivalent to `url_target.to_string()`
pub fn to_url(&self) -> String {
self.to_url_yt_host("https://www.youtube.com")
}
/// Convert the URL target to a YouTube URL with a specified YouTube host.
///
/// Used to redirect to alternative YouTube frontends like Piped or Invidious.
///
/// **Note:** Music album URL targets are still converted to `music.youtube.com/browse/*`,
/// since these URLs are not supported by Piped or Invidious.
pub fn to_url_yt_host(&self, yt_host: &str) -> String {
match self {
UrlTarget::Video { id, start_time, .. } => match start_time {
@ -73,6 +108,7 @@ impl UrlTarget {
}
}
/// Validate the YouTube ID from the URL target
pub(crate) fn validate(&self) -> Result<(), Error> {
match self {
UrlTarget::Video { id, .. } => {
@ -107,11 +143,6 @@ impl UrlTarget {
#PLAYER
*/
pub trait FileFormat {
/// Get the file extension (".xyz") of the file format
fn extension(&self) -> &str;
}
/// Video player data
#[derive(Clone, Debug, Serialize, Deserialize)]
#[non_exhaustive]
@ -172,13 +203,17 @@ pub struct VideoStream {
pub url: String,
/// YouTube stream format identifier
pub itag: u32,
/// Stream bitrate (in bits/second)
pub bitrate: u32,
/// Average stream bitrate (in bits/second)
pub average_bitrate: u32,
/// Video file size in bytes
pub size: Option<u64>,
/// Index range (used for DASH streaming)
pub index_range: Option<Range<u32>>,
/// Init range (used for DASH streaming)
pub init_range: Option<Range<u32>>,
// Video duration in milliseconds
/// Video duration in milliseconds
pub duration_ms: Option<u32>,
/// Video width in pixels
pub width: u32,
@ -209,13 +244,17 @@ pub struct AudioStream {
pub url: String,
/// YouTube stream format identifier
pub itag: u32,
/// Stream bitrate (in bits/second)
pub bitrate: u32,
/// Average stream bitrate (in bits/second)
pub average_bitrate: u32,
/// Audio file size in bytes
pub size: u64,
/// Index range (used for DASH streaming)
pub index_range: Option<Range<u32>>,
/// Init range (used for DASH streaming)
pub init_range: Option<Range<u32>>,
// Audio duration in milliseconds
/// Audio duration in milliseconds
pub duration_ms: Option<u32>,
/// MIME file type
pub mime: String,
@ -241,94 +280,6 @@ pub struct AudioStream {
pub track: Option<AudioTrack>,
}
pub trait YtStream {
fn url(&self) -> &str;
fn itag(&self) -> u32;
fn bitrate(&self) -> u32;
fn averate_bitrate(&self) -> u32;
fn size(&self) -> Option<u64>;
fn index_range(&self) -> Option<Range<u32>>;
fn init_range(&self) -> Option<Range<u32>>;
fn duration_ms(&self) -> Option<u32>;
fn mime(&self) -> &str;
}
impl YtStream for VideoStream {
fn url(&self) -> &str {
&self.url
}
fn itag(&self) -> u32 {
self.itag
}
fn bitrate(&self) -> u32 {
self.bitrate
}
fn averate_bitrate(&self) -> u32 {
self.average_bitrate
}
fn size(&self) -> Option<u64> {
self.size
}
fn index_range(&self) -> Option<Range<u32>> {
self.index_range.clone()
}
fn init_range(&self) -> Option<Range<u32>> {
self.init_range.clone()
}
fn duration_ms(&self) -> Option<u32> {
self.duration_ms
}
fn mime(&self) -> &str {
&self.mime
}
}
impl YtStream for AudioStream {
fn url(&self) -> &str {
&self.url
}
fn itag(&self) -> u32 {
self.itag
}
fn bitrate(&self) -> u32 {
self.bitrate
}
fn averate_bitrate(&self) -> u32 {
self.average_bitrate
}
fn size(&self) -> Option<u64> {
Some(self.size)
}
fn index_range(&self) -> Option<Range<u32>> {
self.index_range.clone()
}
fn init_range(&self) -> Option<Range<u32>> {
self.init_range.clone()
}
fn duration_ms(&self) -> Option<u32> {
self.duration_ms
}
fn mime(&self) -> &str {
&self.mime
}
}
/// Video codec
#[derive(
Default, Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
@ -336,6 +287,7 @@ impl YtStream for AudioStream {
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum VideoCodec {
/// Unknown codec
#[default]
Unknown,
/// MPEG-4 Part 14 <https://en.wikipedia.org/wiki/MPEG-4_Part_14>
@ -355,6 +307,7 @@ pub enum VideoCodec {
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum AudioCodec {
/// Unknown codec
#[default]
Unknown,
/// MP4A aka AAC: <https://en.wikipedia.org/wiki/Advanced_Audio_Coding>
@ -396,16 +349,6 @@ pub struct AudioTrack {
pub is_default: bool,
}
impl FileFormat for VideoFormat {
fn extension(&self) -> &str {
match self {
VideoFormat::ThreeGp => ".3gp",
VideoFormat::Mp4 => ".mp4",
VideoFormat::Webm => ".webm",
}
}
}
/// Audio file type
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "snake_case")]
@ -417,15 +360,6 @@ pub enum AudioFormat {
Webm,
}
impl FileFormat for AudioFormat {
fn extension(&self) -> &str {
match self {
AudioFormat::M4a => ".m4a",
AudioFormat::Webm => ".webm",
}
}
}
/// YouTube provides subtitles in different formats.
///
/// srv1 (XML) is the default format, to request a different format you have
@ -1023,7 +957,9 @@ pub struct ArtistItem {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct ArtistId {
/// Unique YouTube channel ID
pub id: Option<String>,
/// Artist name
pub name: String,
}
@ -1203,10 +1139,12 @@ pub struct MusicSearchResult {
pub corrected_query: Option<String>,
/// Order of the item sections of the search page, starting with
/// the most relevant.
pub order: Vec<MusicEntityType>,
pub order: Vec<MusicItemType>,
}
/// Generic YouTube Music item
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[allow(missing_docs)]
pub enum MusicItem {
Track(TrackItem),
Album(AlbumItem),
@ -1214,18 +1152,21 @@ pub enum MusicItem {
Playlist(MusicPlaylistItem),
}
/// YouTube Music item type
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum MusicEntityType {
#[allow(missing_docs)]
pub enum MusicItemType {
Track,
Album,
Artist,
Playlist,
}
/// Filtered YouTube Music search result
/// Filtered YouTube Music search result (one item type)
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct MusicSearchFiltered<T> {
/// Search items
pub items: Paginator<T>,
/// Corrected search query
///
@ -1239,8 +1180,11 @@ pub struct MusicSearchFiltered<T> {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct TrackDetails {
/// Track metadata
pub track: TrackItem,
/// ID to fetch lyrics
pub lyrics_id: Option<String>,
/// ID to fetch related tracks
pub related_id: Option<String>,
}
@ -1254,7 +1198,7 @@ pub struct Lyrics {
pub footer: String,
}
/// YouTube Music entities related to a track
/// YouTube Music items related to a track
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct MusicRelated {

6
src/model/ordering.rs
View file

@ -4,7 +4,13 @@ use crate::model::AudioCodec;
use super::{AudioStream, VideoStream};
/// Trait for ordering YouTube video/audio streams by quality
///
/// analogous to [`std::cmp::Ord`]
pub trait QualityOrd {
/// Compare two streams by quality
///
/// analogous to [`std::cmp::Ord::cmp`]
fn quality_cmp(&self, other: &Self) -> Ordering;
}

39
src/model/paginator.rs
View file

@ -1,9 +1,9 @@
//! Wrapper model for progressively fetched items
use std::convert::TryInto;
use serde::{Deserialize, Serialize};
use crate::param::ContinuationEndpoint;
/// Wrapper around progressively fetched items
///
/// The paginator is a wrapper around a list of items that are fetched
@ -34,7 +34,7 @@ pub struct Paginator<T> {
#[serde(skip_serializing_if = "Option::is_none")]
pub visitor_data: Option<String>,
/// YouTube API endpoint to fetch continuations from
pub endpoint: ContinuationEndpoint,
pub(crate) endpoint: ContinuationEndpoint,
}
impl<T> Default for Paginator<T> {
@ -49,6 +49,39 @@ impl<T> Default for Paginator<T> {
}
}
/// YouTube API endpoint to fetch continuations from
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
#[allow(missing_docs)]
pub enum ContinuationEndpoint {
Browse,
Search,
Next,
MusicBrowse,
MusicSearch,
MusicNext,
}
impl ContinuationEndpoint {
pub(crate) fn as_str(self) -> &'static str {
match self {
ContinuationEndpoint::Browse | ContinuationEndpoint::MusicBrowse => "browse",
ContinuationEndpoint::Search | ContinuationEndpoint::MusicSearch => "search",
ContinuationEndpoint::Next | ContinuationEndpoint::MusicNext => "next",
}
}
pub(crate) fn is_music(self) -> bool {
matches!(
self,
ContinuationEndpoint::MusicBrowse
| ContinuationEndpoint::MusicSearch
| ContinuationEndpoint::MusicNext
)
}
}
impl<T> Paginator<T> {
pub(crate) fn new(count: Option<u64>, items: Vec<T>, ctoken: Option<String>) -> Self {
Self::new_ext(count, items, ctoken, None, ContinuationEndpoint::Browse)

24
src/model/richtext.rs
View file

@ -6,19 +6,31 @@ use crate::util;
use super::UrlTarget;
/// Text content with links
#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub struct RichText(pub Vec<TextComponent>);
/// Text component forming a [`RichText`] object
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[non_exhaustive]
pub enum TextComponent {
/// Plain text
Text(String),
/// Web link
Web { text: String, url: String },
/// Link to a YouTube entity
YouTube { text: String, target: UrlTarget },
Web {
/// Link text
text: String,
/// Link URL
url: String,
},
/// Link to a YouTube item
YouTube {
/// Link text
text: String,
/// YouTube URL target
target: UrlTarget,
},
}
/// Trait for converting rich text to plain text.
@ -46,6 +58,7 @@ pub trait ToHtml {
}
impl TextComponent {
/// Get the text from the component
pub fn get_text(&self) -> &str {
match self {
TextComponent::Text(text) => text,
@ -54,9 +67,12 @@ impl TextComponent {
}
}
/// Get the link URL from the component
///
/// Returns an empty string if the component is not a link.
pub fn get_url(&self, yt_host: &str) -> String {
match self {
TextComponent::Text(_) => "".to_owned(),
TextComponent::Text(_) => String::new(),
TextComponent::Web { url, .. } => url.to_owned(),
TextComponent::YouTube { target, .. } => target.to_url_yt_host(yt_host),
}

130
src/model/traits.rs Normal file
View file

@ -0,0 +1,130 @@
//! Traits for working with response models
use std::ops::Range;
pub use super::{convert::FromYtItem, ordering::QualityOrd};
use super::{AudioFormat, AudioStream, VideoFormat, VideoStream};
/// Trait for YouTube streams (video and audio)
pub trait YtStream {
/// Stream URL
fn url(&self) -> &str;
/// YouTube stream format identifier
fn itag(&self) -> u32;
/// Stream bitrate (in bits/second)
fn bitrate(&self) -> u32;
/// Average stream bitrate (in bits/second)
fn averate_bitrate(&self) -> u32;
/// File size in bytes
fn size(&self) -> Option<u64>;
/// Index range (used for DASH streaming)
fn index_range(&self) -> Option<Range<u32>>;
/// Init range (used for DASH streaming)
fn init_range(&self) -> Option<Range<u32>>;
/// Stream duration in milliseconds
fn duration_ms(&self) -> Option<u32>;
/// MIME file type
fn mime(&self) -> &str;
}
impl YtStream for VideoStream {
fn url(&self) -> &str {
&self.url
}
fn itag(&self) -> u32 {
self.itag
}
fn bitrate(&self) -> u32 {
self.bitrate
}
fn averate_bitrate(&self) -> u32 {
self.average_bitrate
}
fn size(&self) -> Option<u64> {
self.size
}
fn index_range(&self) -> Option<Range<u32>> {
self.index_range.clone()
}
fn init_range(&self) -> Option<Range<u32>> {
self.init_range.clone()
}
fn duration_ms(&self) -> Option<u32> {
self.duration_ms
}
fn mime(&self) -> &str {
&self.mime
}
}
impl YtStream for AudioStream {
fn url(&self) -> &str {
&self.url
}
fn itag(&self) -> u32 {
self.itag
}
fn bitrate(&self) -> u32 {
self.bitrate
}
fn averate_bitrate(&self) -> u32 {
self.average_bitrate
}
fn size(&self) -> Option<u64> {
Some(self.size)
}
fn index_range(&self) -> Option<Range<u32>> {
self.index_range.clone()
}
fn init_range(&self) -> Option<Range<u32>> {
self.init_range.clone()
}
fn duration_ms(&self) -> Option<u32> {
self.duration_ms
}
fn mime(&self) -> &str {
&self.mime
}
}
/// Trait for file types
pub trait FileFormat {
/// Get the file extension (".xyz") of the file format
fn extension(&self) -> &str;
}
impl FileFormat for VideoFormat {
fn extension(&self) -> &str {
match self {
VideoFormat::ThreeGp => ".3gp",
VideoFormat::Mp4 => ".mp4",
VideoFormat::Webm => ".webm",
}
}
}
impl FileFormat for AudioFormat {
fn extension(&self) -> &str {
match self {
AudioFormat::M4a => ".m4a",
AudioFormat::Webm => ".webm",
}
}
}

35
src/param/mod.rs
View file

@ -1,40 +1,9 @@
//! Query parameters
mod stream_filter;
pub mod locale;
pub mod search_filter;
pub use locale::{Country, Language};
use serde::{Deserialize, Serialize};
pub use stream_filter::StreamFilter;
/// YouTube API endpoint to fetch continuations from
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum ContinuationEndpoint {
Browse,
Search,
Next,
MusicBrowse,
MusicSearch,
MusicNext,
}
impl ContinuationEndpoint {
pub(crate) fn as_str(self) -> &'static str {
match self {
ContinuationEndpoint::Browse | ContinuationEndpoint::MusicBrowse => "browse",
ContinuationEndpoint::Search | ContinuationEndpoint::MusicSearch => "search",
ContinuationEndpoint::Next | ContinuationEndpoint::MusicNext => "next",
}
}
pub(crate) fn is_music(self) -> bool {
matches!(
self,
ContinuationEndpoint::MusicBrowse
| ContinuationEndpoint::MusicSearch
| ContinuationEndpoint::MusicNext
)
}
}

34
src/param/search_filter.rs
View file

@ -1,13 +1,23 @@
//! YouTube search filter
use std::collections::BTreeSet;
use crate::util::{self, ProtoBuilder};
/// YouTube search filter
///
/// Allows you to filter YouTube's search results by
/// item type, features (e.g. HD, 3D, Creative commons), upload date
/// and length.
///
/// Additionally you can sort the search results by rating, upload date
/// or view count.
#[derive(Default, Debug)]
pub struct SearchFilter {
sort: Option<Order>,
features: BTreeSet<Feature>,
date: Option<UploadDate>,
entity: Option<Entity>,
item_type: Option<ItemType>,
length: Option<Length>,
verbatim: bool,
}
@ -61,9 +71,10 @@ pub enum UploadDate {
Year = 5,
}
/// YouTube entity type to filter by
/// YouTube item type to filter by
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Entity {
#[allow(missing_docs)]
pub enum ItemType {
Video = 1,
Channel = 2,
Playlist = 3,
@ -81,6 +92,7 @@ pub enum Length {
}
impl SearchFilter {
/// Get a new [`SearchFilter`]
pub fn new() -> Self {
Self::default()
}
@ -122,14 +134,14 @@ impl SearchFilter {
}
/// Filter videos by entity type
pub fn entity(mut self, entity: Entity) -> Self {
self.entity = Some(entity);
pub fn item_type(mut self, entity: ItemType) -> Self {
self.item_type = Some(entity);
self
}
/// Filter videos by entity type
pub fn entity_opt(mut self, entity: Option<Entity>) -> Self {
self.entity = entity;
pub fn item_type_opt(mut self, entity: Option<ItemType>) -> Self {
self.item_type = entity;
self
}
@ -163,7 +175,7 @@ impl SearchFilter {
if let Some(date) = self.date {
filters.varint(1, date as u64);
}
if let Some(entity) = self.entity {
if let Some(entity) = self.item_type {
filters.varint(2, entity as u64);
}
if let Some(length) = self.length {
@ -200,9 +212,9 @@ mod tests {
use super::*;
#[rstest]
#[case(SearchFilter::new().entity(Entity::Video), "EgIQAQ%253D%253D")]
#[case(SearchFilter::new().entity(Entity::Channel), "EgIQAg%253D%253D")]
#[case(SearchFilter::new().entity(Entity::Playlist), "EgIQAw%253D%253D")]
#[case(SearchFilter::new().item_type(ItemType::Video), "EgIQAQ%253D%253D")]
#[case(SearchFilter::new().item_type(ItemType::Channel), "EgIQAg%253D%253D")]
#[case(SearchFilter::new().item_type(ItemType::Playlist), "EgIQAw%253D%253D")]
#[case(SearchFilter::new().date(UploadDate::Hour), "EgIIAQ%253D%253D")]
#[case(SearchFilter::new().date(UploadDate::Day), "EgIIAg%253D%253D")]
#[case(SearchFilter::new().date(UploadDate::Week), "EgIIAw%253D%253D")]

7
src/param/stream_filter.rs
View file

@ -3,10 +3,11 @@
use std::cmp::Ordering;
use crate::model::{
AudioCodec, AudioFormat, AudioStream, QualityOrd, VideoCodec, VideoFormat, VideoPlayer,
traits::QualityOrd, AudioCodec, AudioFormat, AudioStream, VideoCodec, VideoFormat, VideoPlayer,
VideoStream,
};
/// The StreamFilter is used for selecting audio/video streams from an extracted video
#[derive(Debug, Default, Clone)]
pub struct StreamFilter<'a> {
audio_max_bitrate: Option<u32>,
@ -229,6 +230,7 @@ impl<'a> StreamFilter<'a> {
}
impl VideoPlayer {
/// Select the audio stream which is the best match for the given [`StreamFilter`]
pub fn select_audio_stream(&self, filter: &StreamFilter) -> Option<&AudioStream> {
let mut fallback: Option<&AudioStream> = None;
@ -282,14 +284,17 @@ impl VideoPlayer {
.or(fallback)
}
/// Select the video stream which is the best match for the given [`StreamFilter`]
pub fn select_video_stream(&self, filter: &StreamFilter) -> Option<&VideoStream> {
Self::_select_video_stream(&self.video_streams, filter)
}
/// Select the video-only stream which is the best match for the given [`StreamFilter`]
pub fn select_video_only_stream(&self, filter: &StreamFilter) -> Option<&VideoStream> {
Self::_select_video_stream(&self.video_only_streams, filter)
}
/// Select a video and audio stream which is the best match for the given [`StreamFilter`]
pub fn select_video_audio_stream(
&self,
filter: &StreamFilter,

36
src/report.rs
View file

@ -1,4 +1,20 @@
//! Error reporting
//! # Error reporting
//!
//! Due to the instability of the Innertube API, RustyPipe may not be able to parse
//! every item from every YouTube response. To allow for easy debugging, RustyPipe
//! can create and store error reports.
//!
//! These reports contain information about the RustyPipe client, the performed
//! operation, the request sent to YouTube and the received response data.
//!
//! With the report data the error can be reproduced and RustyPipe can be patched to
//! handle YouTube's changes to the response model.
//!
//! By default, RustyPipe stores the reports as JSON files
//! (e.g `rustypipe_reports/2022-11-05_22-58-59_ERR`).
//!
//! By implementing the [`Reporter`] trait you can handle error reports in other ways
//! (e.g. store them in a database, send them via mail, log to Sentry, etc).
use std::{
collections::BTreeMap,
@ -17,11 +33,13 @@ use crate::{deobfuscate::DeobfData, util};
const FILENAME_FORMAT: &[time::format_description::FormatItem] =
format_description!("[year]-[month]-[day]_[hour]-[minute]-[second]");
/// RustyPipe error report
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Report {
pub info: Info,
/// Report level
/// Information about the RustyPipe client
pub info: RustyPipeInfo,
/// Severity of the report
pub level: Level,
/// RustyPipe operation (e.g. `get_player`)
pub operation: String,
@ -36,9 +54,10 @@ pub struct Report {
pub http_request: HTTPRequest,
}
/// Information about the RustyPipe client
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Info {
pub struct RustyPipeInfo {
/// Rust package name (`rustypipe`)
pub package: String,
/// Package version (`0.1.0`)
@ -48,6 +67,7 @@ pub struct Info {
pub date: OffsetDateTime,
}
/// Reported HTTP request data
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct HTTPRequest {
@ -65,6 +85,7 @@ pub struct HTTPRequest {
pub resp_body: String,
}
/// Severity of the report
#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Level {
/// **Debug**: Operation successful, report generation was forced by setting
@ -76,7 +97,7 @@ pub enum Level {
ERR,
}
impl Default for Info {
impl Default for RustyPipeInfo {
fn default() -> Self {
Self {
package: "rustypipe".to_owned(),
@ -86,15 +107,20 @@ impl Default for Info {
}
}
/// Trait used to abstract the report storage behavior, so you can handle RustyPipe's
/// error reports in your preferred way.
pub trait Reporter: Sync + Send {
/// Store a RustyPipe error report
fn report(&self, report: &Report);
}
/// [`Reporter`] implementation that writes reports as JSON files to the given folder
pub struct FileReporter {
path: PathBuf,
}
impl FileReporter {
/// Create a new reporter that stores error reports in the given folder
pub fn new<P: AsRef<Path>>(path: P) -> Self {
Self {
path: path.as_ref().to_path_buf(),

10
src/timeago.rs
View file

@ -30,7 +30,9 @@ use crate::{
/// Example: "14 hours ago" => `TimeAgo {n: 14, unit: TimeUnit::Hour}`
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct TimeAgo {
/// Number of time units
pub n: u8,
/// Time unit
pub unit: TimeUnit,
}
@ -42,12 +44,20 @@ pub struct TimeAgo {
/// - "2 months ago" => `ParsedDate::Relative(TimeAgo {n: 2, unit: TimeUnit::Month})`
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ParsedDate {
/// Absolute date
///
/// Example: "Jul 2, 2014"
Absolute(Date),
/// Relative date
///
/// Example: "2 months ago"
Relative(TimeAgo),
}
/// Parsed time unit
#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[serde(rename_all = "lowercase")]
#[allow(missing_docs)]
pub enum TimeUnit {
Second,
Minute,

30
tests/youtube.rs
View file

@ -10,9 +10,11 @@ use time::OffsetDateTime;
use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery};
use rustypipe::error::{Error, ExtractionError};
use rustypipe::model::{
richtext::ToPlaintext, AlbumType, AudioCodec, AudioFormat, Channel, FromYtItem,
MusicEntityType, MusicGenre, Paginator, UrlTarget, Verification, VideoCodec, VideoFormat,
YouTubeItem, YtStream,
paginator::Paginator,
richtext::ToPlaintext,
traits::{FromYtItem, YtStream},
AlbumType, AudioCodec, AudioFormat, Channel, MusicGenre, MusicItemType, UrlTarget,
Verification, VideoCodec, VideoFormat, YouTubeItem,
};
use rustypipe::param::{
search_filter::{self, SearchFilter},
@ -1109,15 +1111,15 @@ async fn search() {
}
#[rstest]
#[case::video(search_filter::Entity::Video)]
#[case::channel(search_filter::Entity::Channel)]
#[case::playlist(search_filter::Entity::Playlist)]
#[case::video(search_filter::ItemType::Video)]
#[case::channel(search_filter::ItemType::Channel)]
#[case::playlist(search_filter::ItemType::Playlist)]
#[tokio::test]
async fn search_filter_entity(#[case] entity: search_filter::Entity) {
async fn search_filter_item_type(#[case] item_type: search_filter::ItemType) {
let rp = RustyPipe::builder().strict().build();
let mut result = rp
.query()
.search_filter("with no videos", &SearchFilter::new().entity(entity))
.search_filter("with no videos", &SearchFilter::new().item_type(item_type))
.await
.unwrap();
@ -1126,13 +1128,13 @@ async fn search_filter_entity(#[case] entity: search_filter::Entity) {
result.items.items.iter().for_each(|item| match item {
YouTubeItem::Video(_) => {
assert_eq!(entity, search_filter::Entity::Video);
assert_eq!(item_type, search_filter::ItemType::Video);
}
YouTubeItem::Channel(_) => {
assert_eq!(entity, search_filter::Entity::Channel);
assert_eq!(item_type, search_filter::ItemType::Channel);
}
YouTubeItem::Playlist(_) => {
assert_eq!(entity, search_filter::Entity::Playlist);
assert_eq!(item_type, search_filter::ItemType::Playlist);
}
});
}
@ -1249,7 +1251,7 @@ async fn startpage() {
// The startpage requires visitor data to fetch continuations
assert!(startpage.visitor_data.is_some());
assert_next(startpage, rp.query(), 20, 2).await;
assert_next(startpage, rp.query(), 15, 2).await;
}
#[tokio::test]
@ -1508,7 +1510,7 @@ async fn music_search(#[case] typo: bool) {
assert!(!res.albums.is_empty(), "no albums");
assert!(!res.artists.is_empty(), "no artists");
assert!(!res.playlists.is_empty(), "no playlists");
assert_eq!(res.order[0], MusicEntityType::Track);
assert_eq!(res.order[0], MusicItemType::Track);
if typo {
assert_eq!(res.corrected_query.unwrap(), "black mamba");
@ -2207,7 +2209,7 @@ async fn ab3_search_channel_handles() {
rp.query()
.search_filter(
"test",
&SearchFilter::new().entity(search_filter::Entity::Channel),
&SearchFilter::new().item_type(search_filter::ItemType::Channel),
)
.await
.unwrap();