feat: add starpage continuation

codegen/src/download_testfiles.rs

@@ -30,6 +30,7 @@ pub async fn download_testfiles(project_root: &Path) {
     search_playlists(&testfiles).await;
     search_empty(&testfiles).await;
     startpage(&testfiles).await;
+    startpage_cont(&testfiles).await;
     trending(&testfiles).await;
 }
 
@@ -381,6 +382,21 @@ async fn startpage(testfiles: &Path) {
     rp.query().startpage().await.unwrap();
 }
 
+async fn startpage_cont(testfiles: &Path) {
+    let mut json_path = testfiles.to_path_buf();
+    json_path.push("trends");
+    json_path.push("startpage_cont.json");
+    if json_path.exists() {
+        return;
+    }
+
+    let rp = RustyPipe::new();
+    let startpage = rp.query().startpage().await.unwrap();
+
+    let rp = rp_testfile(&json_path);
+    startpage.next(rp.query()).await.unwrap();
+}
+
 async fn trending(testfiles: &Path) {
     let mut json_path = testfiles.to_path_buf();
     json_path.push("trends");

src/client/mod.rs

@@ -90,6 +90,8 @@ struct ClientInfo {
     platform: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     original_url: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    visitor_data: Option<String>,
     hl: Language,
     gl: Country,
 }
@@ -773,6 +775,7 @@ impl RustyPipeQuery {
                     device_model: None,
                     platform: "DESKTOP".to_owned(),
                     original_url: Some("https://www.youtube.com/".to_owned()),
+                    visitor_data: None,
                     hl,
                     gl,
                 },
@@ -788,6 +791,7 @@ impl RustyPipeQuery {
                     device_model: None,
                     platform: "DESKTOP".to_owned(),
                     original_url: Some("https://music.youtube.com/".to_owned()),
+                    visitor_data: None,
                     hl,
                     gl,
                 },
@@ -803,6 +807,7 @@ impl RustyPipeQuery {
                     device_model: None,
                     platform: "TV".to_owned(),
                     original_url: None,
+                    visitor_data: None,
                     hl,
                     gl,
                 },
@@ -820,6 +825,7 @@ impl RustyPipeQuery {
                     device_model: None,
                     platform: "MOBILE".to_owned(),
                     original_url: None,
+                    visitor_data: None,
                     hl,
                     gl,
                 },
@@ -835,6 +841,7 @@ impl RustyPipeQuery {
                     device_model: Some(IOS_DEVICE_MODEL.to_owned()),
                     platform: "MOBILE".to_owned(),
                     original_url: None,
+                    visitor_data: None,
                     hl,
                     gl,
                 },

src/client/pagination.rs

@@ -1,6 +1,7 @@
 use crate::error::Error;
 use crate::model::{
     ChannelPlaylist, ChannelVideo, Comment, Paginator, PlaylistVideo, RecommendedVideo, SearchItem,
+    SearchVideo,
 };
 
 use super::RustyPipeQuery;
@@ -70,3 +71,4 @@ paginator!(
     RustyPipeQuery::channel_playlists_continuation
 );
 paginator!(SearchItem, RustyPipeQuery::search_continuation);
+paginator!(SearchVideo, RustyPipeQuery::startpage_continuation);

src/client/response/mod.rs

@@ -15,6 +15,7 @@ pub use playlist_music::PlaylistMusic;
 pub use search::Search;
 pub use search::SearchCont;
 pub use trends::Startpage;
+pub use trends::StartpageCont;
 pub use trends::Trending;
 pub use video_details::VideoComments;
 pub use video_details::VideoDetails;

src/client/response/trends.rs

@@ -9,6 +9,16 @@ use super::{ContentRenderer, ContentsRenderer, VideoListItem, VideoRenderer};
 #[serde(rename_all = "camelCase")]
 pub struct Startpage {
     pub contents: Contents<BrowseResultsStartpage>,
+    pub response_context: ResponseContext,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct StartpageCont {
+    #[serde(default)]
+    #[serde_as(as = "VecSkipError<_>")]
+    pub on_response_received_actions: Vec<OnResponseReceivedAction>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -91,6 +101,12 @@ pub struct ShelfContentsRenderer {
     pub items: MapResult<Vec<TrendingListItem>>,
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ResponseContext {
+    pub visitor_data: Option<String>,
+}
+
 #[serde_as]
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -101,3 +117,17 @@ pub enum TrendingListItem {
     #[serde(other, deserialize_with = "ignore_any")]
     None,
 }
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct OnResponseReceivedAction {
+    pub append_continuation_items_action: AppendAction,
+}
+
+#[serde_as]
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AppendAction {
+    #[serde_as(as = "VecLogError<_>")]
+    pub continuation_items: MapResult<Vec<VideoListItem>>,
+}

src/client/trends.rs

@@ -1,13 +1,14 @@
 use crate::{
     error::{Error, ExtractionError},
     model::{Paginator, SearchVideo},
+    param::Language,
     serializer::MapResult,
     util::TryRemove,
 };
 
 use super::{
     response::{self, TryFromWLang},
-    ClientType, MapResponse, QBrowse, RustyPipeQuery,
+    ClientType, MapResponse, QBrowse, QContinuation, RustyPipeQuery,
 };
 
 impl RustyPipeQuery {
@@ -28,6 +29,31 @@ impl RustyPipeQuery {
         .await
     }
 
+    pub async fn startpage_continuation(
+        self,
+        combined_ctoken: &str,
+    ) -> Result<Paginator<SearchVideo>, Error> {
+        let (visitor_data, ctoken) = combined_ctoken
+            .split_once('|')
+            .ok_or_else(|| Error::Other("Invalid ctoken".into()))?;
+
+        let mut context = self.get_context(ClientType::Desktop, true).await;
+        context.client.visitor_data = Some(visitor_data.to_owned());
+        let request_body = QContinuation {
+            context,
+            continuation: ctoken,
+        };
+
+        self.execute_request::<response::StartpageCont, _, _>(
+            ClientType::Desktop,
+            "startpage_continuation",
+            combined_ctoken,
+            "browse",
+            &request_body,
+        )
+        .await
+    }
+
     pub async fn trending(self) -> Result<Vec<SearchVideo>, Error> {
         let context = self.get_context(ClientType::Desktop, true).await;
         let request_body = QBrowse {
@@ -62,35 +88,33 @@ impl MapResponse<Paginator<SearchVideo>> for response::Startpage {
             .rich_grid_renderer
             .contents;
 
-        let mut warnings = grid.warnings;
-        let mut ctoken = None;
-        let items = grid
-            .c
-            .into_iter()
-            .filter_map(|item| match item {
-                response::VideoListItem::RichItemRenderer {
-                    content: response::RichItem::VideoRenderer(video),
-                } => match SearchVideo::from_w_lang(video, lang) {
-                    Ok(video) => Some(video),
-                    Err(e) => {
-                        warnings.push(e.to_string());
-                        None
-                    }
-                },
-                response::VideoListItem::ContinuationItemRenderer {
-                    continuation_endpoint,
-                } => {
-                    ctoken = Some(continuation_endpoint.continuation_command.token);
-                    None
-                }
-                _ => None,
-            })
-            .collect();
+        Ok(map_startpage_videos(
+            grid,
+            lang,
+            self.response_context.visitor_data.as_deref(),
+        ))
+    }
+}
 
-        Ok(MapResult {
-            c: Paginator::new(None, items, ctoken),
-            warnings,
-        })
+impl MapResponse<Paginator<SearchVideo>> for response::StartpageCont {
+    fn map_response(
+        self,
+        id: &str,
+        lang: crate::param::Language,
+        _deobf: Option<&crate::deobfuscate::Deobfuscator>,
+    ) -> Result<MapResult<Paginator<SearchVideo>>, ExtractionError> {
+        let (visitor_data, _) = id
+            .split_once('|')
+            .ok_or_else(|| ExtractionError::InvalidData("Invalid ctoken".into()))?;
+
+        let mut received_actions = self.on_response_received_actions;
+        let items = received_actions
+            .try_swap_remove(0)
+            .ok_or_else(|| ExtractionError::InvalidData("no contents".into()))?
+            .append_continuation_items_action
+            .continuation_items;
+
+        Ok(map_startpage_videos(items, lang, Some(visitor_data)))
     }
 }
 
@@ -146,3 +170,42 @@ impl MapResponse<Vec<SearchVideo>> for response::Trending {
         Ok(MapResult { c: items, warnings })
     }
 }
+
+fn map_startpage_videos(
+    videos: MapResult<Vec<response::VideoListItem>>,
+    lang: Language,
+    visitor_data: Option<&str>,
+) -> MapResult<Paginator<SearchVideo>> {
+    let mut warnings = videos.warnings;
+    let mut ctoken = None;
+    let items = videos
+        .c
+        .into_iter()
+        .filter_map(|item| match item {
+            response::VideoListItem::RichItemRenderer {
+                content: response::RichItem::VideoRenderer(video),
+            } => match SearchVideo::from_w_lang(video, lang) {
+                Ok(video) => Some(video),
+                Err(e) => {
+                    warnings.push(e.to_string());
+                    None
+                }
+            },
+            response::VideoListItem::ContinuationItemRenderer {
+                continuation_endpoint,
+            } => {
+                ctoken = Some(continuation_endpoint.continuation_command.token);
+                None
+            }
+            _ => None,
+        })
+        .collect();
+
+    let combined_ctoken = ctoken
+        .and_then(|ctoken| visitor_data.map(|visitor_data| format!("{}|{}", visitor_data, ctoken)));
+
+    MapResult {
+        c: Paginator::new(None, items, combined_ctoken),
+        warnings,
+    }
+}

tests/youtube.rs

@@ -1220,6 +1220,21 @@ async fn startpage() {
     assert!(!result.is_exhausted());
 }
 
+#[tokio::test]
+async fn startpage_cont() {
+    let rp = RustyPipe::builder().strict().build();
+    let startpage = rp.query().startpage().await.unwrap();
+
+    let next = startpage.next(rp.query()).await.unwrap().unwrap();
+
+    assert!(
+        next.items.len() > 20,
+        "expected > 20 items, got {}",
+        next.items.len()
+    );
+    assert!(!next.is_exhausted());
+}
+
 #[tokio::test]
 async fn trending() {
     let rp = RustyPipe::builder().strict().build();