diff --git a/cli/src/main.rs b/cli/src/main.rs index 44c94eb..d205933 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -8,7 +8,7 @@ use reqwest::{Client, ClientBuilder}; use rustypipe::{ client::RustyPipe, model::{UrlTarget, VideoId}, - param::{search_filter, StreamFilter}, + param::{search_filter, ChannelVideoTab, StreamFilter}, }; use serde::Serialize; @@ -113,6 +113,7 @@ enum ChannelTab { Videos, Shorts, Live, + Playlists, Info, } @@ -564,27 +565,16 @@ async fn main() { print_data(&artist, format, pretty); } else { match tab { - ChannelTab::Videos => { - let mut channel = rp.query().channel_videos(&id).await.unwrap(); - channel - .content - .extend_limit(rp.query(), limit) - .await - .unwrap(); - print_data(&channel, format, pretty); - } - ChannelTab::Shorts => { - let mut channel = rp.query().channel_shorts(&id).await.unwrap(); - channel - .content - .extend_limit(rp.query(), limit) - .await - .unwrap(); - print_data(&channel, format, pretty); - } - ChannelTab::Live => { + ChannelTab::Videos | ChannelTab::Shorts | ChannelTab::Live => { + let video_tab = match tab { + ChannelTab::Videos => ChannelVideoTab::Videos, + ChannelTab::Shorts => ChannelVideoTab::Shorts, + ChannelTab::Live => ChannelVideoTab::Live, + _ => unreachable!(), + }; let mut channel = - rp.query().channel_livestreams(&id).await.unwrap(); + rp.query().channel_videos_tab(&id, video_tab).await.unwrap(); + channel .content .extend_limit(rp.query(), limit) @@ -592,6 +582,10 @@ async fn main() { .unwrap(); print_data(&channel, format, pretty); } + ChannelTab::Playlists => { + let channel = rp.query().channel_playlists(&id).await.unwrap(); + print_data(&channel, format, pretty); + } ChannelTab::Info => { let channel = rp.query().channel_info(&id).await.unwrap(); print_data(&channel, format, pretty); diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index ad18b7c..0e72d08 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -10,7 +10,7 @@ use rustypipe::{ client::{ClientType, RustyPipe}, param::{ search_filter::{self, ItemType, SearchFilter}, - Country, + ChannelVideoTab, Country, }, report::{Report, Reporter}, }; @@ -305,7 +305,7 @@ async fn channel_shorts() { let rp = rp_testfile(&json_path); rp.query() - .channel_shorts("UCh8gHdtzO2tXd593_bjErWg") + .channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts) .await .unwrap(); } @@ -318,7 +318,7 @@ async fn channel_livestreams() { let rp = rp_testfile(&json_path); rp.query() - .channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ") + .channel_videos_tab("UC2DjFE7Xf11URZqWBigcVOQ", ChannelVideoTab::Live) .await .unwrap(); } diff --git a/notes/channel_order.md b/notes/channel_order.md new file mode 100644 index 0000000..1c9b699 --- /dev/null +++ b/notes/channel_order.md @@ -0,0 +1,69 @@ +# Channel order + +Fields: + +- `2:0:string` Channel ID +- `15:0:embedded` Videos tab +- `10:0:embedded` Shorts tab +- `14:0:embedded` Livestreams tab +- `2:0:string`: targetId for YouTube's web framework (`"\n$"` + any UUID) +- `3:1:varint` Sort order (1: Latest, 2: Popular) + +Popular videos + +```json +{ + "80226972:0:embedded": { + "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw", + "3:1:base64": { + "110:0:embedded": { + "3:0:embedded": { + "15:0:embedded": { + "2:0:string": "\n$6461d7c8-0000-2040-87aa-089e0827e420", + "3:1:varint": 2 + } + } + } + } + } +} +``` + +Popular shorts +```json +{ + "80226972:0:embedded": { + "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw", + "3:1:base64": { + "110:0:embedded": { + "3:0:embedded": { + "10:0:embedded": { + "2:0:string": "\n$64679ffb-0000-26b3-a1bd-582429d2c794", + "3:1:varint": 2 + } + } + } + } + } +} +``` + +Popular streams + +```json +{ + "80226972:0:embedded": { + "2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw", + "3:1:base64": { + "110:0:embedded": { + "3:0:embedded": { + "14:0:embedded": { + "2:0:string": "\n$64693069-0000-2a1e-8c7d-582429bd5ba8", + "3:1:varint": 2 + } + } + } + } + } +} +``` diff --git a/src/client/channel.rs b/src/client/channel.rs index 0580f2f..0044f3a 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -1,14 +1,15 @@ -use std::borrow::Cow; - use serde::Serialize; use url::Url; use crate::{ error::{Error, ExtractionError}, - model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem}, - param::Language, + model::{ + paginator::{ContinuationEndpoint, Paginator}, + Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem, + }, + param::{ChannelOrder, ChannelVideoTab, Language}, serializer::MapResult, - util, + util::{self, ProtoBuilder}, }; use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; @@ -18,13 +19,13 @@ use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext}; struct QChannel<'a> { context: YTContext<'a>, browse_id: &'a str, - params: Params, + params: ChannelTab, #[serde(skip_serializing_if = "Option::is_none")] query: Option<&'a str>, } #[derive(Debug, Serialize)] -enum Params { +enum ChannelTab { #[serde(rename = "EgZ2aWRlb3PyBgQKAjoA")] Videos, #[serde(rename = "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")] @@ -39,11 +40,21 @@ enum Params { Search, } +impl From for ChannelTab { + fn from(value: ChannelVideoTab) -> Self { + match value { + ChannelVideoTab::Videos => Self::Videos, + ChannelVideoTab::Shorts => Self::Shorts, + ChannelVideoTab::Live => Self::Live, + } + } +} + impl RustyPipeQuery { async fn _channel_videos>( &self, channel_id: S, - params: Params, + params: ChannelTab, query: Option<&str>, operation: &str, ) -> Result>, Error> { @@ -71,28 +82,54 @@ impl RustyPipeQuery { &self, channel_id: S, ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Videos, None, "channel_videos") + self._channel_videos(channel_id, ChannelTab::Videos, None, "channel_videos") .await } - /// Get the short videos from a YouTube channel - pub async fn channel_shorts>( + /// Get a ordered list of videos from a YouTube channel + /// + /// This function does not return channel metadata. + pub async fn channel_videos_order>( &self, channel_id: S, - ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Shorts, None, "channel_shorts") + order: ChannelOrder, + ) -> Result, Error> { + self.channel_videos_tab_order(channel_id, ChannelVideoTab::Videos, order) .await } - /// Get the livestreams from a YouTube channel - pub async fn channel_livestreams>( + /// Get the specified video tab from a YouTube channel + pub async fn channel_videos_tab>( &self, channel_id: S, + tab: ChannelVideoTab, ) -> Result>, Error> { - self._channel_videos(channel_id, Params::Live, None, "channel_livestreams") + self._channel_videos(channel_id, tab.into(), None, "channel_videos") .await } + /// Get a ordered list of videos from the specified tab of a YouTube channel + /// + /// This function does not return channel metadata. + pub async fn channel_videos_tab_order>( + &self, + channel_id: S, + tab: ChannelVideoTab, + order: ChannelOrder, + ) -> Result, Error> { + let visitor_data = match tab { + ChannelVideoTab::Shorts => Some(self.get_visitor_data().await?), + _ => None, + }; + + self.continuation( + order_ctoken(channel_id.as_ref(), tab, order), + ContinuationEndpoint::Browse, + visitor_data.as_deref(), + ) + .await + } + /// Search the videos of a channel pub async fn channel_search, S2: AsRef>( &self, @@ -101,7 +138,7 @@ impl RustyPipeQuery { ) -> Result>, Error> { self._channel_videos( channel_id, - Params::Search, + ChannelTab::Search, Some(query.as_ref()), "channel_search", ) @@ -118,7 +155,7 @@ impl RustyPipeQuery { let request_body = QChannel { context, browse_id: channel_id, - params: Params::Playlists, + params: ChannelTab::Playlists, query: None, }; @@ -142,7 +179,7 @@ impl RustyPipeQuery { let request_body = QChannel { context, browse_id: channel_id, - params: Params::Info, + params: ChannelTab::Info, query: None, }; @@ -451,16 +488,16 @@ fn map_channel_content( .or(tab.tab_renderer.content.section_list_renderer) }); - let content = match channel_content { - Some(list) => list.contents, - None => { - // YouTube may show the "Featured" tab if the requested tab is empty/does not exist - if featured_tab { - MapResult::default() - } else { - return Err(ExtractionError::InvalidData(Cow::Borrowed( - "could not extract content", - ))); + // YouTube may show the "Featured" tab if the requested tab is empty/does not exist + let content = if featured_tab { + MapResult::default() + } else { + match channel_content { + Some(list) => list.contents, + None => { + return Err(ExtractionError::InvalidData( + "could not extract content".into(), + )) } } }; @@ -495,6 +532,47 @@ fn combine_channel_data(channel_data: Channel<()>, content: T) -> Channel } } +/// Get the continuation token to fetch channel videos in the given order +fn order_ctoken(channel_id: &str, tab: ChannelVideoTab, order: ChannelOrder) -> String { + _order_ctoken( + channel_id, + tab, + order, + &format!("\n${}", util::random_uuid()), + ) +} + +/// Get the continuation token to fetch channel videos in the given order +/// (fixed targetId for testing) +fn _order_ctoken( + channel_id: &str, + tab: ChannelVideoTab, + order: ChannelOrder, + target_id: &str, +) -> String { + let mut pb_tab = ProtoBuilder::new(); + pb_tab.string(2, target_id); + pb_tab.varint(3, order as u64); + + let mut pb_3 = ProtoBuilder::new(); + pb_3.embedded(tab.order_ctoken_id(), pb_tab); + + let mut pb_110 = ProtoBuilder::new(); + pb_110.embedded(3, pb_3); + + let mut pbi = ProtoBuilder::new(); + pbi.embedded(110, pb_110); + + let mut pb_80226972 = ProtoBuilder::new(); + pb_80226972.string(2, channel_id); + pb_80226972.string(3, &pbi.to_base64()); + + let mut pb = ProtoBuilder::new(); + pb.embedded(80226972, pb_80226972); + + pb.to_base64() +} + #[cfg(test)] mod tests { use std::{fs::File, io::BufReader}; @@ -505,11 +583,13 @@ mod tests { use crate::{ client::{response, MapResponse}, model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem}, - param::Language, + param::{ChannelOrder, ChannelVideoTab, Language}, serializer::MapResult, util::tests::TESTFILES, }; + use super::_order_ctoken; + #[rstest] #[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")] #[case::music("videos_music", "UC_vmjW5e1xEHhYjY2a0kK1A")] @@ -585,4 +665,33 @@ mod tests { ); insta::assert_ron_snapshot!("map_channel_info", map_res.c); } + + #[test] + fn order_ctoken() { + let channel_id = "UCXuqSBlHAE6Xw-yeJA0Tunw"; + + let videos_popular_token = _order_ctoken( + channel_id, + ChannelVideoTab::Videos, + ChannelOrder::Popular, + "\n$6461d7c8-0000-2040-87aa-089e0827e420", + ); + assert_eq!(videos_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXg2S2hJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBZyUzRCUzRA%3D%3D"); + + let shorts_popular_token = _order_ctoken( + channel_id, + ChannelVideoTab::Shorts, + ChannelOrder::Popular, + "\n$64679ffb-0000-26b3-a1bd-582429d2c794", + ); + assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUVlBZyUzRCUzRA%3D%3D"); + + let live_popular_token = _order_ctoken( + channel_id, + ChannelVideoTab::Live, + ChannelOrder::Popular, + "\n$64693069-0000-2a1e-8c7d-582429bd5ba8", + ); + assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ1lBZyUzRCUzRA%3D%3D"); + } } diff --git a/src/client/mod.rs b/src/client/mod.rs index d76f5b5..edc130e 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -191,6 +191,7 @@ const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+"; const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/"; const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/"; const YOUTUBE_MUSIC_V1_URL: &str = "https://music.youtube.com/youtubei/v1/"; +const YOUTUBE_HOME_URL: &str = "https://www.youtube.com/"; const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/"; const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false"; @@ -644,7 +645,7 @@ impl RustyPipe { self.extract_client_version( Some("https://www.youtube.com/sw.js"), "https://www.youtube.com/results?search_query=", - "https://www.youtube.com", + YOUTUBE_HOME_URL, None, ) .await @@ -654,8 +655,8 @@ impl RustyPipe { async fn extract_music_client_version(&self) -> Result { self.extract_client_version( Some("https://music.youtube.com/sw.js"), - "https://music.youtube.com", - "https://music.youtube.com", + YOUTUBE_MUSIC_HOME_URL, + YOUTUBE_MUSIC_HOME_URL, None, ) .await @@ -812,7 +813,7 @@ impl RustyPipe { } } - async fn get_ytm_visitor_data(&self) -> Result { + async fn get_visitor_data(&self) -> Result { log::debug!("getting YTM visitor data"); let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?; @@ -903,7 +904,7 @@ impl RustyPipeQuery { client_name: "WEB", client_version: Cow::Owned(self.client.get_desktop_client_version().await), platform: "DESKTOP", - original_url: Some("https://www.youtube.com/"), + original_url: Some(YOUTUBE_HOME_URL), visitor_data, hl, gl, @@ -918,7 +919,7 @@ impl RustyPipeQuery { client_name: "WEB_REMIX", client_version: Cow::Owned(self.client.get_music_client_version().await), platform: "DESKTOP", - original_url: Some("https://music.youtube.com/"), + original_url: Some(YOUTUBE_MUSIC_HOME_URL), visitor_data, hl, gl, @@ -942,7 +943,7 @@ impl RustyPipeQuery { request: Some(RequestYT::default()), user: User::default(), third_party: Some(ThirdParty { - embed_url: "https://www.youtube.com/", + embed_url: YOUTUBE_HOME_URL, }), }, ClientType::Android => YTContext { @@ -993,8 +994,8 @@ impl RustyPipeQuery { .post(format!( "{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") + .header(header::ORIGIN, YOUTUBE_HOME_URL) + .header(header::REFERER, YOUTUBE_HOME_URL) .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "1") .header( @@ -1008,8 +1009,8 @@ impl RustyPipeQuery { .post(format!( "{YOUTUBE_MUSIC_V1_URL}{endpoint}?key={DESKTOP_MUSIC_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) - .header(header::ORIGIN, "https://music.youtube.com") - .header(header::REFERER, "https://music.youtube.com") + .header(header::ORIGIN, YOUTUBE_MUSIC_HOME_URL) + .header(header::REFERER, YOUTUBE_MUSIC_HOME_URL) .header(header::COOKIE, self.client.inner.consent_cookie.to_owned()) .header("X-YouTube-Client-Name", "67") .header( @@ -1023,8 +1024,8 @@ impl RustyPipeQuery { .post(format!( "{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}" )) - .header(header::ORIGIN, "https://www.youtube.com") - .header(header::REFERER, "https://www.youtube.com") + .header(header::ORIGIN, YOUTUBE_HOME_URL) + .header(header::REFERER, YOUTUBE_HOME_URL) .header("X-YouTube-Client-Name", "1") .header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION), ClientType::Android => self @@ -1060,11 +1061,11 @@ impl RustyPipeQuery { } } - /// Get a YouTube Music visitor data cookie, which is necessary for certain requests - async fn get_ytm_visitor_data(&self) -> Result { + /// Get a YouTube visitor data cookie, which is necessary for certain requests + async fn get_visitor_data(&self) -> Result { match &self.opts.visitor_data { Some(vd) => Ok(vd.to_owned()), - None => self.client.get_ytm_visitor_data().await, + None => self.client.get_visitor_data().await, } } @@ -1304,9 +1305,9 @@ mod tests { } #[test] - fn t_get_ytm_visitor_data() { + fn t_get_visitor_data() { let rp = RustyPipe::new(); - let visitor_data = tokio_test::block_on(rp.get_ytm_visitor_data()).unwrap(); + let visitor_data = tokio_test::block_on(rp.get_visitor_data()).unwrap(); assert!(visitor_data.ends_with("%3D")); assert_eq!(visitor_data.len(), 32) } diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index 2c8ae8a..8724dd0 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -27,7 +27,7 @@ impl RustyPipeQuery { ) -> Result { let artist_id = artist_id.as_ref(); let visitor_data = match all_albums { - true => Some(self.get_ytm_visitor_data().await?), + true => Some(self.get_visitor_data().await?), false => None, }; diff --git a/src/client/music_details.rs b/src/client/music_details.rs index 0d35eae..342675c 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -109,7 +109,7 @@ impl RustyPipeQuery { radio_id: S, ) -> Result, Error> { let radio_id = radio_id.as_ref(); - let visitor_data = self.get_ytm_visitor_data().await?; + let visitor_data = self.get_visitor_data().await?; let context = self .get_context(ClientType::DesktopMusic, true, Some(&visitor_data)) .await; diff --git a/src/client/pagination.rs b/src/client/pagination.rs index bfb07cd..6874a21 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -102,8 +102,12 @@ impl MapResponse> for response::Continuation { .and_then(|actions| { actions .into_iter() - .next() .map(|action| action.append_continuation_items_action.continuation_items) + .reduce(|mut acc, mut items| { + acc.c.append(&mut items.c); + acc.warnings.append(&mut items.warnings); + acc + }) }) .or_else(|| { self.continuation_contents diff --git a/src/client/player.rs b/src/client/player.rs index 367821b..da1bc0d 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -492,8 +492,6 @@ fn map_audio_stream( deobf: &Deobfuscator, last_nsig: &mut [String; 2], ) -> MapResult> { - static LANG_PATTERN: Lazy = Lazy::new(|| Regex::new(r#"^([a-z]{2,3})\."#).unwrap()); - let (mtype, codecs) = match parse_mime(&f.mime_type) { Some(x) => x, None => { @@ -535,18 +533,12 @@ fn map_audio_stream( loudness_db: f.loudness_db, throttled, track: match f.audio_track { - Some(t) => { - let lang = LANG_PATTERN - .captures(&t.id) - .map(|m| m.get(1).unwrap().as_str().to_owned()); - - Some(AudioTrack { - id: t.id, - lang, - lang_name: t.display_name, - is_default: t.audio_is_default, - }) - } + Some(t) => Some(AudioTrack { + lang: t.id.split('.').next().map(str::to_owned), + id: t.id, + lang_name: t.display_name, + is_default: t.audio_is_default, + }), None => None, }, }), diff --git a/src/client/response/mod.rs b/src/client/response/mod.rs index b321dcc..a682f95 100644 --- a/src/client/response/mod.rs +++ b/src/client/response/mod.rs @@ -219,6 +219,7 @@ pub(crate) struct Continuation { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct ContinuationActionWrap { + #[serde(alias = "reloadContinuationItemsCommand")] pub append_continuation_items_action: ContinuationAction, } diff --git a/src/client/response/video_item.rs b/src/client/response/video_item.rs index b812f2d..dba27be 100644 --- a/src/client/response/video_item.rs +++ b/src/client/response/video_item.rs @@ -1,5 +1,3 @@ -use once_cell::sync::Lazy; -use regex::Regex; use serde::Deserialize; use serde_with::{ json::JsonString, rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError, @@ -430,11 +428,17 @@ impl YouTubeListMapper { } fn map_video(&mut self, video: VideoRenderer) -> VideoItem { - let mut tn_overlays = video.thumbnail_overlays; + let is_live = video.thumbnail_overlays.is_live() || video.badges.is_live(); + let is_short = video.thumbnail_overlays.is_short(); + let length_text = video.length_text.or_else(|| { - tn_overlays - .try_swap_remove(0) - .map(|overlay| overlay.thumbnail_overlay_time_status_renderer.text) + video + .thumbnail_overlays + .into_iter() + .find(|ol| { + ol.thumbnail_overlay_time_status_renderer.style == TimeOverlayStyle::Default + }) + .map(|ol| ol.thumbnail_overlay_time_status_renderer.text) }); VideoItem { @@ -472,8 +476,8 @@ impl YouTubeListMapper { view_count: video .view_count_text .map(|txt| util::parse_numeric(&txt).unwrap_or_default()), - is_live: tn_overlays.is_live() || video.badges.is_live(), - is_short: tn_overlays.is_short(), + is_live, + is_short, is_upcoming: video.upcoming_event_data.is_some(), short_description: video .detailed_metadata_snippets @@ -483,9 +487,6 @@ impl YouTubeListMapper { } fn map_short_video(&mut self, video: ReelItemRenderer, lang: Language) -> VideoItem { - static ACCESSIBILITY_SEP_REGEX: Lazy = - Lazy::new(|| Regex::new(" [-\u{2013}] ").unwrap()); - let pub_date_txt = video.navigation_endpoint.map(|n| { n.reel_watch_endpoint .overlay @@ -499,7 +500,7 @@ impl YouTubeListMapper { id: video.video_id, name: video.headline, length: video.accessibility.and_then(|acc| { - ACCESSIBILITY_SEP_REGEX.split(&acc).nth(1).and_then(|s| { + acc.rsplit(" - ").nth(1).and_then(|s| { timeago::parse_video_duration_or_warn(self.lang, s, &mut self.warnings) }) }), diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap index 835d9ff..834d87c 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap @@ -168,7 +168,7 @@ Channel( publish_date: "[date]", publish_date_txt: None, view_count: Some(94), - is_live: false, + is_live: true, is_short: false, is_upcoming: false, short_description: None, @@ -209,7 +209,7 @@ Channel( publish_date: "[date]", publish_date_txt: None, view_count: Some(381), - is_live: false, + is_live: true, is_short: false, is_upcoming: false, short_description: None, @@ -414,7 +414,7 @@ Channel( publish_date: "[date]", publish_date_txt: None, view_count: Some(2043), - is_live: false, + is_live: true, is_short: false, is_upcoming: false, short_description: None, @@ -783,7 +783,7 @@ Channel( publish_date: "[date]", publish_date_txt: None, view_count: Some(4030), - is_live: false, + is_live: true, is_short: false, is_upcoming: false, short_description: None, diff --git a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap index 9912aec..0f8a1cb 100644 --- a/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap +++ b/src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap @@ -141,7 +141,7 @@ Channel( publish_date_txt: Some("1 day ago"), view_count: Some(443549), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -167,7 +167,7 @@ Channel( publish_date_txt: Some("2 days ago"), view_count: Some(1154962), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -234,7 +234,7 @@ Channel( publish_date_txt: Some("6 days ago"), view_count: Some(1388173), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -260,7 +260,7 @@ Channel( publish_date_txt: Some("7 days ago"), view_count: Some(1738301), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -286,7 +286,7 @@ Channel( publish_date_txt: Some("9 days ago"), view_count: Some(1316594), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -353,7 +353,7 @@ Channel( publish_date_txt: Some("11 days ago"), view_count: Some(1412213), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -379,7 +379,7 @@ Channel( publish_date_txt: Some("13 days ago"), view_count: Some(1513305), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -405,7 +405,7 @@ Channel( publish_date_txt: Some("2 weeks ago"), view_count: Some(8936223), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -472,7 +472,7 @@ Channel( publish_date_txt: Some("2 weeks ago"), view_count: Some(2769717), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -539,7 +539,7 @@ Channel( publish_date_txt: Some("3 weeks ago"), view_count: Some(572107), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -565,7 +565,7 @@ Channel( publish_date_txt: Some("3 weeks ago"), view_count: Some(1707132), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -591,7 +591,7 @@ Channel( publish_date_txt: Some("3 weeks ago"), view_count: Some(933094), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -617,7 +617,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(5985184), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -643,7 +643,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(14741387), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -669,7 +669,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(2511322), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -695,7 +695,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(2364408), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -762,7 +762,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(1947627), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -788,7 +788,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(4763839), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -814,7 +814,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(1915695), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -840,7 +840,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(7268944), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -866,7 +866,7 @@ Channel( publish_date_txt: Some("1 month ago"), view_count: Some(2539103), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -892,7 +892,7 @@ Channel( publish_date_txt: Some("2 months ago"), view_count: Some(5545680), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -918,7 +918,7 @@ Channel( publish_date_txt: Some("2 months ago"), view_count: Some(2202314), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), @@ -985,7 +985,7 @@ Channel( publish_date_txt: Some("2 months ago"), view_count: Some(6443699), is_live: false, - is_short: false, + is_short: true, is_upcoming: false, short_description: None, ), diff --git a/src/model/mod.rs b/src/model/mod.rs index 9d60374..1632333 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -850,7 +850,7 @@ pub struct VideoItem { pub publish_date: Option, /// Textual video publish date (e.g. `11 months ago`, depends on language) /// - /// Is [`None`] for livestreams. + /// Is [`None`] for livestreams and upcoming videos. pub publish_date_txt: Option, /// View count /// diff --git a/src/model/paginator.rs b/src/model/paginator.rs index 6ca5790..2139ade 100644 --- a/src/model/paginator.rs +++ b/src/model/paginator.rs @@ -34,7 +34,7 @@ pub struct Paginator { #[serde(skip_serializing_if = "Option::is_none")] pub visitor_data: Option, /// YouTube API endpoint to fetch continuations from - pub(crate) endpoint: ContinuationEndpoint, + pub endpoint: ContinuationEndpoint, } impl Default for Paginator { diff --git a/src/param/mod.rs b/src/param/mod.rs index a3f4708..804c7d8 100644 --- a/src/param/mod.rs +++ b/src/param/mod.rs @@ -7,3 +7,34 @@ pub mod search_filter; pub use locale::{Country, Language}; pub use stream_filter::StreamFilter; + +/// Channel video tab +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChannelVideoTab { + /// Regular videos + Videos, + /// Short videos + Shorts, + /// Livestreams + Live, +} + +/// Sort order for channel videos +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChannelOrder { + /// Order videos with the latest upload date first (default) + Latest = 1, + /// Order videos with the highest number of views first + Popular = 2, +} + +impl ChannelVideoTab { + /// Get the tab ID used to create ordered continuation tokens + pub(crate) const fn order_ctoken_id(&self) -> u32 { + match self { + ChannelVideoTab::Videos => 15, + ChannelVideoTab::Shorts => 10, + ChannelVideoTab::Live => 14, + } + } +} diff --git a/src/param/search_filter.rs b/src/param/search_filter.rs index 84a8034..489ae63 100644 --- a/src/param/search_filter.rs +++ b/src/param/search_filter.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; -use crate::util::{self, ProtoBuilder}; +use crate::util::ProtoBuilder; /// YouTube search filter /// @@ -200,8 +200,7 @@ impl SearchFilter { pb.embedded(8, extras) } - let b64 = util::b64_encode(pb.bytes); - urlencoding::encode(&b64).to_string() + pb.to_base64() } } diff --git a/src/util/mod.rs b/src/util/mod.rs index a91375f..e794681 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -83,6 +83,18 @@ pub fn generate_content_playback_nonce() -> String { random_string(CONTENT_PLAYBACK_NONCE_ALPHABET, 16) } +pub fn random_uuid() -> String { + let mut rng = rand::thread_rng(); + format!( + "{:08x}-{:04x}-{:04x}-{:04x}-{:012x}", + rng.gen::(), + rng.gen::(), + rng.gen::(), + rng.gen::(), + rng.gen::() & 0xffffffffffff, + ) +} + /// Split an URL into its base string and parameter map /// /// Example: diff --git a/src/util/protobuf.rs b/src/util/protobuf.rs index 64488ac..8c1b0d6 100644 --- a/src/util/protobuf.rs +++ b/src/util/protobuf.rs @@ -45,6 +45,13 @@ impl ProtoBuilder { self._varint(val); } + /// Write a string field + pub fn string(&mut self, field: u32, string: &str) { + self._field(field, 2); + self._varint(string.len() as u64); + self.bytes.extend_from_slice(string.as_bytes()); + } + /// Write an embedded message /// /// Requires passing another [`ProtoBuilder`] with the embedded message. @@ -53,6 +60,12 @@ impl ProtoBuilder { self._varint(pb.bytes.len() as u64); self.bytes.append(&mut pb.bytes); } + + /// Base64 + urlencode the protobuf data + pub fn to_base64(&self) -> String { + let b64 = super::b64_encode(&self.bytes); + urlencoding::encode(&b64).to_string() + } } fn parse_varint>(pb: &mut P) -> Option { @@ -124,11 +137,6 @@ mod tests { use super::*; - // #[test] - // fn t_parse_varint() { - - // } - #[test] fn t_parse_proto() { let p = "GhhVQzl2cnZOU0wzeGNXR1NrVjg2UkVCU2c%3D"; diff --git a/tests/youtube.rs b/tests/youtube.rs index 928206d..fd2f368 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use rstest::{fixture, rstest}; use rustypipe::model::paginator::ContinuationEndpoint; -use rustypipe::param::Language; +use rustypipe::param::{ChannelOrder, ChannelVideoTab, Language}; use rustypipe::validate; use time::macros::date; use time::OffsetDateTime; @@ -261,15 +261,10 @@ fn get_player( let langs = player_data .audio_streams .iter() - .filter_map(|stream| { - stream - .track - .as_ref() - .map(|t| t.lang.as_ref().unwrap().to_owned()) - }) + .filter_map(|stream| stream.track.as_ref().map(|t| t.lang.as_deref().unwrap())) .collect::>(); - for l in ["en", "es", "fr", "pt", "ru"] { + for l in ["en-US", "es", "fr", "pt", "ru"] { assert!(langs.contains(l), "missing lang: {l}"); } } @@ -779,8 +774,11 @@ fn channel_videos(rp: RustyPipe) { #[rstest] fn channel_shorts(rp: RustyPipe) { - let channel = - tokio_test::block_on(rp.query().channel_shorts("UCh8gHdtzO2tXd593_bjErWg")).unwrap(); + let channel = tokio_test::block_on( + rp.query() + .channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts), + ) + .unwrap(); // dbg!(&channel); assert_eq!(channel.id, "UCh8gHdtzO2tXd593_bjErWg"); @@ -809,8 +807,11 @@ fn channel_shorts(rp: RustyPipe) { #[rstest] fn channel_livestreams(rp: RustyPipe) { - let channel = - tokio_test::block_on(rp.query().channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")).unwrap(); + let channel = tokio_test::block_on( + rp.query() + .channel_videos_tab("UC2DjFE7Xf11URZqWBigcVOQ", ChannelVideoTab::Live), + ) + .unwrap(); // dbg!(&channel); assert_channel_eevblog(&channel); @@ -955,6 +956,63 @@ fn channel_more( assert_channel(&channel_info, id, name, unlocalized || name_unlocalized); } +#[rstest] +#[case::videos("UCcdwLMPsaU2ezNSJU1nFoBQ", ChannelVideoTab::Videos, "XqZsoesa55w")] +#[case::shorts("UCcdwLMPsaU2ezNSJU1nFoBQ", ChannelVideoTab::Shorts, "k91vRvXGwHs")] +#[case::live("UCvqRdlKsE5Q8mf8YXbdIJLw", ChannelVideoTab::Live, "ojes5ULOqhc")] +fn channel_order( + #[case] id: &str, + #[case] tab: ChannelVideoTab, + #[case] most_popular: &str, + rp: RustyPipe, +) { + let latest = tokio_test::block_on(rp.query().channel_videos_tab_order( + id, + tab, + ChannelOrder::Latest, + )) + .unwrap(); + // Upload dates should be in descending order + if tab != ChannelVideoTab::Shorts { + let mut latest_items = latest.items.iter().peekable(); + while let (Some(v), Some(next_v)) = (latest_items.next(), latest_items.peek()) { + if !v.is_upcoming && !v.is_live && !next_v.is_upcoming && !next_v.is_live { + assert_gte( + v.publish_date.unwrap(), + next_v.publish_date.unwrap(), + "latest video date", + ); + } + } + } + assert_next(latest, rp.query(), 15, 2); + + let popular = tokio_test::block_on(rp.query().channel_videos_tab_order( + id, + tab, + ChannelOrder::Popular, + )) + .unwrap(); + // Most popular video should be in top 5 + assert!( + popular.items.iter().take(5).any(|v| v.id == most_popular), + "most popular video {most_popular} not found" + ); + + // View counts should be in descending order + if tab != ChannelVideoTab::Shorts { + let mut popular_items = popular.items.iter().peekable(); + while let (Some(v), Some(next_v)) = (popular_items.next(), popular_items.peek()) { + assert_gte( + v.view_count.unwrap(), + next_v.view_count.unwrap(), + "most popular view count", + ); + } + } + assert_next(popular, rp.query(), 15, 2); +} + #[rstest] #[case::not_exist("UCOpNcN46UbXVtpKMrmU4Abx")] #[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg")] @@ -972,6 +1030,19 @@ fn channel_not_found(#[case] id: &str, rp: RustyPipe) { ); } +#[rstest] +#[case::shorts(ChannelVideoTab::Shorts)] +#[case::live(ChannelVideoTab::Live)] +fn channel_tab_not_found(#[case] tab: ChannelVideoTab, rp: RustyPipe) { + let channel = tokio_test::block_on( + rp.query() + .channel_videos_tab("UCGiJh0NZ52wRhYKYnuZI08Q", tab), + ) + .unwrap(); + + assert!(channel.content.is_empty(), "got: {:?}", channel.content); +} + //#CHANNEL_RSS #[cfg(feature = "rss")] @@ -2240,7 +2311,7 @@ fn assert_approx(left: f64, right: f64) { /// Assert that number A is greater than or equal to number B fn assert_gte(a: T, b: T, msg: &str) { - assert!(a >= b, "expected {b} {msg}, got {a}"); + assert!(a >= b, "expected >= {b} {msg}, got {a}"); } /// Assert that the paginator produces at least n pages