feat: add channel_livestreams and channel_shorts tabs

codegen/src/download_testfiles.rs

@@ -22,6 +22,8 @@ pub async fn download_testfiles(project_root: &Path) {
     comments_latest(&testfiles).await;
     recommendations(&testfiles).await;
     channel_videos(&testfiles).await;
+    channel_shorts(&testfiles).await;
+    channel_livestreams(&testfiles).await;
     channel_playlists(&testfiles).await;
     channel_info(&testfiles).await;
     channel_videos_cont(&testfiles).await;
@@ -258,6 +260,36 @@ async fn channel_videos(testfiles: &Path) {
     }
 }
 
+async fn channel_shorts(testfiles: &Path) {
+    let mut json_path = testfiles.to_path_buf();
+    json_path.push("channel");
+    json_path.push("channel_shorts.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query()
+        .channel_shorts("UCh8gHdtzO2tXd593_bjErWg")
+        .await
+        .unwrap();
+}
+
+async fn channel_livestreams(testfiles: &Path) {
+    let mut json_path = testfiles.to_path_buf();
+    json_path.push("channel");
+    json_path.push("channel_livestreams.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = rp_testfile(&json_path);
+    rp.query()
+        .channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")
+        .await
+        .unwrap();
+}
+
 async fn channel_playlists(testfiles: &Path) {
     let mut json_path = testfiles.to_path_buf();
     json_path.push("channel");

src/client/channel.rs

@@ -30,48 +30,33 @@ struct QChannel<'a> {
 enum Params {
     #[serde(rename = "EgZ2aWRlb3PyBgQKAjoA")]
     Videos,
+    #[serde(rename = "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")]
+    Shorts,
+    #[serde(rename = "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")]
+    Live,
     #[serde(rename = "EglwbGF5bGlzdHMgAQ%3D%3D")]
     Playlists,
     #[serde(rename = "EgVhYm91dPIGBAoCEgA%3D")]
     Info,
 }
 
-#[derive(Debug, Clone, Copy)]
-enum ChannelTab {
-    Videos,
-    Shorts,
-    Live,
-    Playlists,
-    Info,
-}
-
-impl ChannelTab {
-    const fn url_suffix(self) -> &'static str {
-        match self {
-            ChannelTab::Videos => "/videos",
-            ChannelTab::Shorts => "/shorts",
-            ChannelTab::Live => "/streams",
-            ChannelTab::Playlists => "/playlists",
-            ChannelTab::Info => "/about",
-        }
-    }
-}
-
 impl RustyPipeQuery {
-    pub async fn channel_videos(
+    async fn _channel_videos(
         &self,
         channel_id: &str,
+        params: Params,
+        operation: &str,
     ) -> Result<Channel<Paginator<VideoItem>>, Error> {
         let context = self.get_context(ClientType::Desktop, true, None).await;
         let request_body = QChannel {
             context,
             browse_id: channel_id,
-            params: Params::Videos,
+            params,
         };
 
         self.execute_request::<response::Channel, _, _>(
             ClientType::Desktop,
-            "channel_videos",
+            operation,
             channel_id,
             "browse",
             &request_body,
@@ -79,6 +64,30 @@ impl RustyPipeQuery {
         .await
     }
 
+    pub async fn channel_videos(
+        &self,
+        channel_id: &str,
+    ) -> Result<Channel<Paginator<VideoItem>>, Error> {
+        self._channel_videos(channel_id, Params::Videos, "channel_videos")
+            .await
+    }
+
+    pub async fn channel_shorts(
+        &self,
+        channel_id: &str,
+    ) -> Result<Channel<Paginator<VideoItem>>, Error> {
+        self._channel_videos(channel_id, Params::Shorts, "channel_shorts")
+            .await
+    }
+
+    pub async fn channel_livestreams(
+        &self,
+        channel_id: &str,
+    ) -> Result<Channel<Paginator<VideoItem>>, Error> {
+        self._channel_videos(channel_id, Params::Live, "channel_livestreams")
+            .await
+    }
+
     pub async fn channel_playlists(
         &self,
         channel_id: &str,
@@ -126,7 +135,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
         lang: Language,
         _deobf: Option<&crate::deobfuscate::Deobfuscator>,
     ) -> Result<MapResult<Channel<Paginator<VideoItem>>>, ExtractionError> {
-        let content = map_channel_content(self.contents, ChannelTab::Videos, self.alerts)?;
+        let content = map_channel_content(self.contents, self.alerts)?;
         let grid = match content.content {
             response::channel::ChannelContent::GridRenderer { items } => Some(items),
             _ => None,
@@ -160,7 +169,7 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
         lang: Language,
         _deobf: Option<&crate::deobfuscate::Deobfuscator>,
     ) -> Result<MapResult<Channel<Paginator<PlaylistItem>>>, ExtractionError> {
-        let content = map_channel_content(self.contents, ChannelTab::Playlists, self.alerts)?;
+        let content = map_channel_content(self.contents, self.alerts)?;
         let grid = match content.content {
             response::channel::ChannelContent::GridRenderer { items } => Some(items),
             _ => None,
@@ -196,7 +205,7 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
         lang: Language,
         _deobf: Option<&crate::deobfuscate::Deobfuscator>,
     ) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
-        let content = map_channel_content(self.contents, ChannelTab::Info, self.alerts)?;
+        let content = map_channel_content(self.contents, self.alerts)?;
         let mut warnings = Vec::new();
         let meta = match content.content {
             response::channel::ChannelContent::ChannelAboutFullMetadataRenderer(meta) => Some(meta),
@@ -403,7 +412,6 @@ struct MappedChannelContent {
 
 fn map_channel_content(
     contents: Option<response::channel::Contents>,
-    channel_tab: ChannelTab,
     alerts: Option<Vec<response::Alert>>,
 ) -> Result<MappedChannelContent, ExtractionError> {
     match contents {
@@ -434,13 +442,9 @@ fn map_channel_content(
                         || tab.tab_renderer.content.rich_grid_renderer.is_some())
                 {
                     featured_tab = true;
-                } else if cmp_url_suffix(
+                } else if cmp_url_suffix(&tab.tab_renderer.endpoint, "/shorts") {
-                    &tab.tab_renderer.endpoint,
-                    ChannelTab::Shorts.url_suffix(),
-                ) {
                     has_shorts = true;
-                } else if cmp_url_suffix(&tab.tab_renderer.endpoint, ChannelTab::Live.url_suffix())
+                } else if cmp_url_suffix(&tab.tab_renderer.endpoint, "/streams") {
-                {
                     has_live = true;
                 }
             }
@@ -448,22 +452,18 @@ fn map_channel_content(
             let channel_content = tabs
                 .into_iter()
                 .filter_map(|tab| {
-                    if cmp_url_suffix(&tab.tab_renderer.endpoint, channel_tab.url_suffix()) {
+                    let content = tab.tab_renderer.content;
-                        let content = tab.tab_renderer.content;
+                    match (content.rich_grid_renderer, content.section_list_renderer) {
-                        match (content.rich_grid_renderer, content.section_list_renderer) {
+                        (Some(rich_grid), _) => Some(ChannelContent::GridRenderer {
-                            (Some(rich_grid), _) => Some(ChannelContent::GridRenderer {
+                            items: rich_grid.contents,
-                                items: rich_grid.contents,
+                        }),
-                            }),
+                        (None, Some(section_list)) => {
-                            (None, Some(section_list)) => {
+                            let mut contents = section_list.contents;
-                                let mut contents = section_list.contents;
+                            contents.try_swap_remove(0).and_then(|mut i| {
-                                contents.try_swap_remove(0).and_then(|mut i| {
+                                i.item_section_renderer.contents.try_swap_remove(0)
-                                    i.item_section_renderer.contents.try_swap_remove(0)
+                            })
-                                })
-                            }
-                            (None, None) => None,
                         }
-                    } else {
+                        (None, None) => None,
-                        None
                     }
                 })
                 .next();
@@ -506,16 +506,18 @@ mod tests {
     };
 
     #[rstest]
-    #[case::base("base", "UC2DjFE7Xf11URZqWBigcVOQ")]
+    #[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")]
-    #[case::music("music", "UC_vmjW5e1xEHhYjY2a0kK1A")]
+    #[case::music("videos_music", "UC_vmjW5e1xEHhYjY2a0kK1A")]
+    #[case::withshorts("videos_shorts", "UCh8gHdtzO2tXd593_bjErWg")]
+    #[case::live("videos_live", "UChs0pSaEoNLV4mevBFGaoKA")]
+    #[case::empty("videos_empty", "UCxBa895m48H5idw5li7h-0g")]
+    #[case::upcoming("videos_upcoming", "UCcvfHa-GHSOHFAjU0-Ie57A")]
+    #[case::richgrid("videos_20221011_richgrid", "UCh8gHdtzO2tXd593_bjErWg")]
+    #[case::richgrid2("videos_20221011_richgrid2", "UC2DjFE7Xf11URZqWBigcVOQ")]
     #[case::shorts("shorts", "UCh8gHdtzO2tXd593_bjErWg")]
-    #[case::live("live", "UChs0pSaEoNLV4mevBFGaoKA")]
+    #[case::livestreams("livestreams", "UC2DjFE7Xf11URZqWBigcVOQ")]
-    #[case::empty("empty", "UCxBa895m48H5idw5li7h-0g")]
-    #[case::upcoming("upcoming", "UCcvfHa-GHSOHFAjU0-Ie57A")]
-    #[case::richgrid("20221011_richgrid", "UCh8gHdtzO2tXd593_bjErWg")]
-    #[case::richgrid2("20221011_richgrid2", "UC2DjFE7Xf11URZqWBigcVOQ")]
     fn map_channel_videos(#[case] name: &str, #[case] id: &str) {
-        let filename = format!("testfiles/channel/channel_videos_{}.json", name);
+        let filename = format!("testfiles/channel/channel_{}.json", name);
         let json_path = Path::new(&filename);
         let json_file = File::open(json_path).unwrap();
 
@@ -530,12 +532,12 @@ mod tests {
             map_res.warnings
         );
 
-        if name == "upcoming" {
+        if name == "videos_upcoming" {
-            insta::assert_ron_snapshot!(format!("map_channel_videos_{}", name), map_res.c, {
+            insta::assert_ron_snapshot!(format!("map_channel_{}", name), map_res.c, {
                 ".content.items[1:].publish_date" => "[date]",
             });
         } else {
-            insta::assert_ron_snapshot!(format!("map_channel_videos_{}", name), map_res.c, {
+            insta::assert_ron_snapshot!(format!("map_channel_{}", name), map_res.c, {
                 ".content.items[].publish_date" => "[date]",
             });
         }

src/client/response/video_item.rs

@@ -1,6 +1,8 @@
+use fancy_regex::Regex;
+use once_cell::sync::Lazy;
 use serde::Deserialize;
 use serde_with::{json::JsonString, serde_as, DefaultOnError, VecSkipError};
-use time::OffsetDateTime;
+use time::{Duration, OffsetDateTime};
 
 use super::{ChannelBadge, ContinuationEndpoint, Thumbnails};
 use crate::{
@@ -8,7 +10,7 @@ use crate::{
     param::Language,
     serializer::{
         ignore_any,
-        text::{Text, TextComponent},
+        text::{AccessibilityText, Text, TextComponent},
         MapResult, VecLogError,
     },
     timeago,
@@ -21,6 +23,7 @@ use crate::{
 pub(crate) enum YouTubeListItem {
     #[serde(alias = "gridVideoRenderer", alias = "compactVideoRenderer")]
     VideoRenderer(VideoRenderer),
+    ReelItemRenderer(ReelItemRenderer),
 
     #[serde(alias = "gridPlaylistRenderer")]
     PlaylistRenderer(PlaylistRenderer),
@@ -98,6 +101,7 @@ pub(crate) struct VideoRenderer {
     #[serde_as(as = "VecSkipError<_>")]
     pub badges: Vec<VideoBadge>,
     /// Contains Short/Live tag
+    #[serde(default)]
     #[serde_as(as = "VecSkipError<_>")]
     pub thumbnail_overlays: Vec<TimeOverlay>,
     /// Abbreviated video description (on startpage)
@@ -110,6 +114,27 @@ pub(crate) struct VideoRenderer {
     pub upcoming_event_data: Option<UpcomingEventData>,
 }
 
+/// Short video item
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct ReelItemRenderer {
+    pub video_id: String,
+    pub thumbnail: Thumbnails,
+    #[serde_as(as = "Text")]
+    pub headline: String,
+    /// Contains `No views` if the view count is zero
+    #[serde_as(as = "Option<Text>")]
+    pub view_count_text: Option<String>,
+    /// video duration
+    ///
+    /// Example: `the horror maze - 44 seconds - play video`
+    ///
+    /// Dashes may be `\u2013` (emdash)
+    #[serde_as(as = "Option<AccessibilityText>")]
+    pub accessibility: Option<String>,
+}
+
 /// Playlist displayed in search results
 #[serde_as]
 #[derive(Debug, Deserialize)]
@@ -363,6 +388,39 @@ impl<T> YouTubeListMapper<T> {
         }
     }
 
+    fn map_short_video(&self, video: ReelItemRenderer) -> VideoItem {
+        static ACCESSIBILITY_SEP_REGEX: Lazy<Regex> =
+            Lazy::new(|| Regex::new(" [-\u{2013}] (.+) [-\u{2013}] ").unwrap());
+
+        VideoItem {
+            id: video.video_id,
+            title: video.headline,
+            length: video.accessibility.and_then(|acc| {
+                ACCESSIBILITY_SEP_REGEX
+                    .captures(&acc)
+                    .ok()
+                    .flatten()
+                    .and_then(|cap| {
+                        cap.get(1).and_then(|c| {
+                            timeago::parse_timeago(self.lang, c.as_str())
+                                .map(|ta| Duration::from(ta).whole_seconds() as u32)
+                        })
+                    })
+            }),
+            thumbnail: video.thumbnail.into(),
+            channel: None,
+            publish_date: None,
+            publish_date_txt: None,
+            view_count: video
+                .view_count_text
+                .map(|txt| util::parse_numeric(&txt).unwrap_or_default()),
+            is_live: false,
+            is_short: true,
+            is_upcoming: false,
+            short_description: None,
+        }
+    }
+
     fn map_playlist(playlist: PlaylistRenderer) -> PlaylistItem {
         PlaylistItem {
             id: playlist.playlist_id,
@@ -413,6 +471,10 @@ impl YouTubeListMapper<YouTubeItem> {
             YouTubeListItem::VideoRenderer(video) => {
                 self.items.push(YouTubeItem::Video(self.map_video(video)));
             }
+            YouTubeListItem::ReelItemRenderer(video) => {
+                self.items
+                    .push(YouTubeItem::Video(self.map_short_video(video)));
+            }
             YouTubeListItem::PlaylistRenderer(playlist) => self
                 .items
                 .push(YouTubeItem::Playlist(Self::map_playlist(playlist))),
@@ -449,6 +511,9 @@ impl YouTubeListMapper<VideoItem> {
             YouTubeListItem::VideoRenderer(video) => {
                 self.items.push(self.map_video(video));
             }
+            YouTubeListItem::ReelItemRenderer(video) => {
+                self.items.push(self.map_short_video(video));
+            }
             YouTubeListItem::ContinuationItemRenderer {
                 continuation_endpoint,
             } => self.ctoken = Some(continuation_endpoint.continuation_command.token),

src/timeago.rs

@@ -82,17 +82,27 @@ impl Mul<u8> for TimeAgo {
     }
 }
 
+impl From<TimeAgo> for Duration {
+    fn from(ta: TimeAgo) -> Self {
+        match ta.unit {
+            TimeUnit::Second => Duration::seconds(ta.n as i64),
+            TimeUnit::Minute => Duration::minutes(ta.n as i64),
+            TimeUnit::Hour => Duration::hours(ta.n as i64),
+            TimeUnit::Day => Duration::days(ta.n as i64),
+            TimeUnit::Week => Duration::weeks(ta.n as i64),
+            TimeUnit::Month => Duration::days(ta.n as i64 * 30),
+            TimeUnit::Year => Duration::days(ta.n as i64 * 365),
+        }
+    }
+}
+
 impl From<TimeAgo> for OffsetDateTime {
     fn from(ta: TimeAgo) -> Self {
         let ts = util::now_sec();
         match ta.unit {
-            TimeUnit::Second => ts - Duration::seconds(ta.n as i64),
-            TimeUnit::Minute => ts - Duration::minutes(ta.n as i64),
-            TimeUnit::Hour => ts - Duration::hours(ta.n as i64),
-            TimeUnit::Day => ts - Duration::days(ta.n as i64),
-            TimeUnit::Week => ts - Duration::weeks(ta.n as i64),
             TimeUnit::Month => ts.replace_date(util::shift_months(ts.date(), -(ta.n as i32))),
             TimeUnit::Year => ts.replace_date(util::shift_years(ts.date(), -(ta.n as i32))),
+            _ => ts - Duration::from(ta),
         }
     }
 }

tests/youtube.rs

@@ -12,6 +12,8 @@ use rustypipe::model::{
 };
 use rustypipe::param::search_filter::{self, SearchFilter};
 
+const VISITOR_DATA_3TAB_CHANNEL_LAYOUT: &str = "CgtOa256ckVkcG5YVSiirbyaBg%3D%3D";
+
 //#PLAYER
 
 #[rstest]
@@ -908,6 +910,75 @@ async fn channel_videos() {
     );
 }
 
+#[tokio::test]
+async fn channel_shorts() {
+    let rp = RustyPipe::builder()
+        .strict()
+        .visitor_data(VISITOR_DATA_3TAB_CHANNEL_LAYOUT)
+        .build();
+    let channel = rp
+        .query()
+        .channel_shorts("UCh8gHdtzO2tXd593_bjErWg")
+        .await
+        .unwrap();
+
+    // dbg!(&channel);
+    assert_eq!(channel.id, "UCh8gHdtzO2tXd593_bjErWg");
+    assert_eq!(channel.name, "Doobydobap");
+    assert!(
+        channel.subscriber_count.unwrap() > 2800000,
+        "expected >2.8M subscribers, got {}",
+        channel.subscriber_count.unwrap()
+    );
+    assert!(!channel.avatar.is_empty(), "got no thumbnails");
+    assert_eq!(channel.verification, Verification::Verified);
+    assert!(channel
+        .description
+        .contains("Hi, I\u{2019}m Tina, aka Doobydobap"));
+    assert_eq!(
+        channel.vanity_url.as_ref().unwrap(),
+        "https://www.youtube.com/c/Doobydobap"
+    );
+    assert!(!channel.banner.is_empty(), "got no banners");
+    assert!(!channel.mobile_banner.is_empty(), "got no mobile banners");
+    assert!(!channel.tv_banner.is_empty(), "got no tv banners");
+
+    assert!(
+        !channel.content.items.is_empty() && !channel.content.is_exhausted(),
+        "got no shorts"
+    );
+
+    let next = channel.content.next(&rp.query()).await.unwrap().unwrap();
+    assert!(
+        !next.is_exhausted() && !next.items.is_empty(),
+        "no more shorts"
+    );
+}
+
+#[tokio::test]
+async fn channel_livestreams() {
+    let rp = RustyPipe::builder()
+        .visitor_data(VISITOR_DATA_3TAB_CHANNEL_LAYOUT)
+        .strict()
+        .build();
+    let channel = rp
+        .query()
+        .channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")
+        .await
+        .unwrap();
+
+    // dbg!(&channel);
+    assert_channel_eevblog(&channel);
+
+    assert!(
+        !channel.content.items.is_empty() && !channel.content.is_exhausted(),
+        "got no streams"
+    );
+
+    let next = channel.content.next(&rp.query()).await.unwrap().unwrap();
+    assert!(!next.items.is_empty(), "no more streams");
+}
+
 #[tokio::test]
 async fn channel_playlists() {
     let rp = RustyPipe::builder().strict().build();