diff --git a/src/youtube/channel.rs b/src/youtube/channel.rs index cb5df07..37a7b37 100644 --- a/src/youtube/channel.rs +++ b/src/youtube/channel.rs @@ -21,6 +21,7 @@ use crate::newpipe::NewPipe; use crate::stream::StreamInfoItem; use crate::youtube::client_request::build_desktop_envelope; use crate::youtube::constants::*; +use crate::youtube::continuation::{continuation_items, token_from_items, ContinuationPage}; use crate::youtube::linkhandler::channel::ChannelIdentifier; use crate::youtube::parsing::{web_client_version, youtube_post_headers}; @@ -109,6 +110,63 @@ pub fn fetch_channel_browse(channel_id: &str) -> Result Result { + let body = fetch_continuation_browse(token)?; + Ok(parse_channel_continuation(&body)) +} + +/// POST `browse` with `{"continuation": token}` (no browseId/params). +/// The response carries appended grid items under +/// `onResponseReceivedActions[]` rather than the tabbed layout of the +/// initial browse. +fn fetch_continuation_browse(token: &str) -> Result { + let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; + let localization = NewPipe::preferred_localization(); + let content_country = NewPipe::preferred_content_country(); + let mut envelope = + build_desktop_envelope(&localization, &content_country, &web_client_version()); + if let Value::Object(ref mut map) = envelope { + map.insert("continuation".into(), Value::String(token.into())); + } + let url = format!("{YOUTUBEI_V1_URL}browse{DISABLE_PRETTY_PRINT_PARAM}"); + let body = serde_json::to_vec(&envelope).map_err(|e| { + ExtractionError::Parsing(ParsingError::Invalid(format!("serialize: {e}"))) + })?; + let mut builder = Request::post(&url, body); + for (k, v) in youtube_post_headers() { + builder = builder.add_header(&k, &v); + } + let resp = downloader.execute(builder.build())?; + if resp.response_code() != 200 { + return Err(ExtractionError::Network(NetworkError::Transport(format!( + "browse continuation HTTP {}", + resp.response_code() + )))); + } + serde_json::from_str(resp.response_body()) + .map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string()))) +} + +/// Parse a channel Videos-tab continuation response. Items arrive as the +/// same `richItemRenderer` cells the initial tab uses, so we reuse +/// `parse_rich_grid_item`; the next token sits at the tail of the same +/// `continuationItems` array. +pub fn parse_channel_continuation(body: &Value) -> ContinuationPage { + let Some(items) = continuation_items(body, "onResponseReceivedActions") else { + return ContinuationPage::default(); + }; + let videos = items.iter().filter_map(parse_rich_grid_item).collect(); + ContinuationPage { + items: videos, + continuation: token_from_items(items), + } +} + fn fetch_browse(channel_id: &str, params: Option<&str>) -> Result { let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; let localization = NewPipe::preferred_localization(); @@ -220,83 +278,60 @@ pub fn parse_channel_browse(channel_id: &str, body: &Value) -> ChannelInfo { /// `lockupViewModel` items (YT migrated channel-videos UI to /// lockupViewModel around 2024). fn parse_videos_tab(body: &Value) -> Vec { - let mut out = Vec::new(); - let tabs = body - .get("contents") - .and_then(|c| c.get("twoColumnBrowseResultsRenderer")) - .and_then(|c| c.get("tabs")) - .and_then(|t| t.as_array()); - let Some(tabs) = tabs else { return out }; - - for tab in tabs { - let Some(tr) = tab.get("tabRenderer") else { continue }; - if !tr - .get("selected") - .and_then(|s| s.as_bool()) - .unwrap_or(false) - { - continue; - } - let Some(items) = tr - .get("content") - .and_then(|c| c.get("richGridRenderer")) - .and_then(|g| g.get("contents")) - .and_then(|c| c.as_array()) - else { - continue; - }; - for cell in items { - // richItemRenderer carries either videoRenderer (legacy) or - // lockupViewModel (current 2026 YT). - let Some(content) = cell - .get("richItemRenderer") - .and_then(|r| r.get("content")) - else { - continue; - }; - if let Some(vr) = content.get("videoRenderer") { - if let Some(item) = crate::youtube::search_extractor::renderer_helpers::video_renderer_to_item(vr) { - out.push(item); - } - } else if let Some(lvm) = content.get("lockupViewModel") { - if let Some(item) = parse_lockup_video(lvm) { - out.push(item); - } - } - } + match selected_tab_grid_contents(body) { + Some(items) => items.iter().filter_map(parse_rich_grid_item).collect(), + None => Vec::new(), } - out } -fn parse_videos_continuation(body: &Value) -> Option { +/// The `contents[]` array of the selected channel tab's +/// `richGridRenderer` (the Videos grid). Shared by the initial-tab parse +/// and the page-1 continuation-token scan so both walk the same path. +fn selected_tab_grid_contents(body: &Value) -> Option<&Vec> { let tabs = body .get("contents") .and_then(|c| c.get("twoColumnBrowseResultsRenderer")) .and_then(|c| c.get("tabs")) .and_then(|t| t.as_array())?; - for tab in tabs { - let Some(tr) = tab.get("tabRenderer") else { continue }; + tabs.iter().find_map(|tab| { + let tr = tab.get("tabRenderer")?; if !tr.get("selected").and_then(|s| s.as_bool()).unwrap_or(false) { - continue; + return None; } - let items = tr - .get("content") + tr.get("content") .and_then(|c| c.get("richGridRenderer")) .and_then(|g| g.get("contents")) - .and_then(|c| c.as_array())?; - for cell in items { - if let Some(token) = cell - .get("continuationItemRenderer") - .and_then(|s| s.get("continuationEndpoint")) - .and_then(|c| c.get("continuationCommand")) - .and_then(|c| c.get("token")) - .and_then(|t| t.as_str()) - { - return Some(token.to_string()); - } - } + .and_then(|c| c.as_array()) + }) +} + +/// Shape one Videos-grid cell into a StreamInfoItem. Handles BOTH legacy +/// `videoRenderer` items and current `lockupViewModel` items (YT migrated +/// channel-videos UI to lockupViewModel around 2024). Used by the initial +/// Videos-tab parse AND `parse_channel_continuation`, which receives the +/// same cell shape under `continuationItems[]`. +/// +/// The common shape is `richItemRenderer.content.{videoRenderer | +/// lockupViewModel}`, but some continuation pages surface the renderer +/// BARE (no richItemRenderer wrapper) — fall back to the cell itself so a +/// layout variant can't make a page silently return zero items while the +/// continuation token keeps advancing. +fn parse_rich_grid_item(cell: &Value) -> Option { + let content = cell + .get("richItemRenderer") + .and_then(|r| r.get("content")) + .unwrap_or(cell); + if let Some(vr) = content.get("videoRenderer") { + crate::youtube::search_extractor::renderer_helpers::video_renderer_to_item(vr) + } else if let Some(lvm) = content.get("lockupViewModel") { + parse_lockup_video(lvm) + } else { + None } - None +} + +fn parse_videos_continuation(body: &Value) -> Option { + token_from_items(selected_tab_grid_contents(body)?) } fn parse_lockup_video(lvm: &Value) -> Option { @@ -591,4 +626,83 @@ mod tests { let info = parse_channel_browse("UCxxx", &body); assert_eq!(info.name, "@SomeChannel"); } + + #[test] + fn channel_continuation_lockup_items_plus_token() { + // The Videos-tab continuation comes back under + // onResponseReceivedActions[0].appendContinuationItemsAction, with + // the same richItemRenderer/lockupViewModel cells as page 1, then a + // trailing continuationItemRenderer for the page after. + let body = json!({ + "onResponseReceivedActions": [{ + "appendContinuationItemsAction": {"continuationItems": [ + {"richItemRenderer": {"content": {"lockupViewModel": { + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "contentId": "abcdefghijk", + "metadata": {"lockupMetadataViewModel": { + "title": {"content": "Page Two Video"} + }} + }}}}, + {"continuationItemRenderer": {"continuationEndpoint": { + "continuationCommand": {"token": "CHANNEL_TOKEN_2"} + }}} + ]} + }] + }); + let page = parse_channel_continuation(&body); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].name, "Page Two Video"); + assert_eq!(page.items[0].url, "https://www.youtube.com/watch?v=abcdefghijk"); + assert_eq!(page.continuation.as_deref(), Some("CHANNEL_TOKEN_2")); + } + + #[test] + fn channel_continuation_legacy_video_renderer() { + let body = json!({ + "onResponseReceivedActions": [{ + "appendContinuationItemsAction": {"continuationItems": [ + {"richItemRenderer": {"content": {"videoRenderer": { + "videoId": "vvvvvvvvvvv", + "title": {"runs": [{"text": "Legacy Grid Item"}]} + }}}} + ]} + }] + }); + let page = parse_channel_continuation(&body); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].name, "Legacy Grid Item"); + // No continuationItemRenderer present → this was the last page. + assert_eq!(page.continuation, None); + } + + #[test] + fn channel_continuation_empty_body_is_empty_page() { + let page = parse_channel_continuation(&json!({})); + assert!(page.items.is_empty()); + assert_eq!(page.continuation, None); + } + + #[test] + fn channel_continuation_bare_lockup_no_rich_wrapper() { + // Defensive: a continuation cell that ships the lockupViewModel + // BARE (no richItemRenderer wrapper) must still parse, or the page + // returns empty while the token keeps advancing. + let body = json!({ + "onResponseReceivedActions": [{ + "appendContinuationItemsAction": {"continuationItems": [ + {"lockupViewModel": { + "contentType": "LOCKUP_CONTENT_TYPE_VIDEO", + "contentId": "barelockup1", + "metadata": {"lockupMetadataViewModel": { + "title": {"content": "Bare Lockup Video"} + }} + }} + ]} + }] + }); + let page = parse_channel_continuation(&body); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].name, "Bare Lockup Video"); + assert_eq!(page.continuation, None); + } } diff --git a/src/youtube/continuation.rs b/src/youtube/continuation.rs new file mode 100644 index 0000000..3fc3132 --- /dev/null +++ b/src/youtube/continuation.rs @@ -0,0 +1,126 @@ +// Shared continuation plumbing for the paginated InnerTube surfaces +// (search results, channel Videos tab). Mirrors how NPE models a +// `Page` / `InfoItemsPage`: a chunk of items plus an opaque token that +// fetches the NEXT chunk. +// +// The two callers (search_extractor + channel) differ only in: +// * which endpoint the continuation POST hits (`search` vs `browse`) +// * which top-level key carries the appended items +// (`onResponseReceivedCommands` for search, +// `onResponseReceivedActions` for browse/channel) +// * how each continuationItem cell is shaped into a StreamInfoItem +// Everything else — the envelope `{"continuation": token}`, the +// `appendContinuationItemsAction.continuationItems[]` unwrap, and the +// trailing `continuationItemRenderer` token scan — is identical, so it +// lives here. + +use serde_json::Value; + +use crate::stream::StreamInfoItem; + +/// A page of stream items plus the token for the next page. +/// `continuation == None` means YT returned no further continuation — +/// i.e. this was the last page. +#[derive(Clone, Debug, Default)] +pub struct ContinuationPage { + pub items: Vec, + pub continuation: Option, +} + +/// Unwrap the `continuationItems[]` array from a continuation response. +/// +/// `action_container` is the top-level key the items live under — +/// `"onResponseReceivedCommands"` for search, `"onResponseReceivedActions"` +/// for browse/channel. Inside, YT wraps the items in either an +/// `appendContinuationItemsAction` (the common case) or, on some +/// first-continuation responses, a `reloadContinuationItemsCommand`. +/// Both carry the same `continuationItems` array, so we accept either. +pub(crate) fn continuation_items<'a>( + body: &'a Value, + action_container: &str, +) -> Option<&'a Vec> { + let actions = body.get(action_container)?.as_array()?; + actions.iter().find_map(|action| { + action + .get("appendContinuationItemsAction") + .or_else(|| action.get("reloadContinuationItemsCommand")) + .and_then(|a| a.get("continuationItems")) + .and_then(|c| c.as_array()) + }) +} + +/// Find the next-page token in a `continuationItems` / grid-`contents` +/// array. YT appends a single `continuationItemRenderer` as the last +/// element when more results exist; absent it, this is the last page. +/// We scan from the tail since that's where it always sits, but the +/// search is order-independent for robustness. +pub(crate) fn token_from_items(items: &[Value]) -> Option { + items.iter().rev().find_map(|cell| { + cell.get("continuationItemRenderer") + .and_then(|c| c.get("continuationEndpoint")) + .and_then(|c| c.get("continuationCommand")) + .and_then(|c| c.get("token")) + .and_then(|t| t.as_str()) + .map(String::from) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn token_scans_trailing_continuation_item() { + let items = vec![ + json!({"itemSectionRenderer": {"contents": []}}), + json!({"continuationItemRenderer": {"continuationEndpoint": { + "continuationCommand": {"token": "NEXT_PAGE_TOKEN"} + }}}), + ]; + assert_eq!(token_from_items(&items).as_deref(), Some("NEXT_PAGE_TOKEN")); + } + + #[test] + fn token_absent_is_last_page() { + let items = vec![json!({"itemSectionRenderer": {"contents": []}})]; + assert_eq!(token_from_items(&items), None); + } + + #[test] + fn unwraps_append_action_for_search_key() { + let body = json!({ + "onResponseReceivedCommands": [ + {"appendContinuationItemsAction": {"continuationItems": [ + {"itemSectionRenderer": {"contents": []}} + ]}} + ] + }); + let items = continuation_items(&body, "onResponseReceivedCommands").unwrap(); + assert_eq!(items.len(), 1); + } + + #[test] + fn unwraps_reload_command_fallback_for_browse_key() { + let body = json!({ + "onResponseReceivedActions": [ + {"reloadContinuationItemsCommand": {"continuationItems": [ + {"richItemRenderer": {}}, {"richItemRenderer": {}} + ]}} + ] + }); + let items = continuation_items(&body, "onResponseReceivedActions").unwrap(); + assert_eq!(items.len(), 2); + } + + #[test] + fn wrong_container_key_returns_none() { + let body = json!({ + "onResponseReceivedActions": [ + {"appendContinuationItemsAction": {"continuationItems": []}} + ] + }); + // search key against a browse-shaped body → None + assert!(continuation_items(&body, "onResponseReceivedCommands").is_none()); + } +} diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 0081c2c..7cd93c4 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -5,6 +5,7 @@ pub mod channel; pub mod client_request; pub mod constants; +pub mod continuation; pub mod itag; pub mod js; pub mod linkhandler; diff --git a/src/youtube/search_extractor.rs b/src/youtube/search_extractor.rs index 8a9dc29..2fc5db0 100644 --- a/src/youtube/search_extractor.rs +++ b/src/youtube/search_extractor.rs @@ -27,6 +27,7 @@ use crate::newpipe::NewPipe; use crate::stream::{StreamInfoItem, StreamType}; use crate::youtube::client_request::build_desktop_envelope; use crate::youtube::constants::*; +use crate::youtube::continuation::{continuation_items, token_from_items, ContinuationPage}; use crate::youtube::linkhandler::search::SearchFilter; use crate::youtube::parsing::{web_client_version, youtube_post_headers}; @@ -71,6 +72,76 @@ pub fn search(query: &str, filter: SearchFilter) -> Result Result { + let downloader = NewPipe::downloader().ok_or(ExtractionError::DownloaderMissing)?; + let localization = NewPipe::preferred_localization(); + let content_country = NewPipe::preferred_content_country(); + + let mut envelope = + build_desktop_envelope(&localization, &content_country, &web_client_version()); + if let Value::Object(ref mut map) = envelope { + // Continuation requests carry only the token — no query/params. + map.insert("continuation".into(), Value::String(token.into())); + } + + let url = format!("{YOUTUBEI_V1_URL}search{DISABLE_PRETTY_PRINT_PARAM}"); + let body = serde_json::to_vec(&envelope).map_err(|e| { + ExtractionError::Parsing(ParsingError::Invalid(format!("serialize: {e}"))) + })?; + let mut builder = Request::post(&url, body); + for (k, v) in youtube_post_headers() { + builder = builder.add_header(&k, &v); + } + let resp = downloader.execute(builder.build())?; + if resp.response_code() != 200 { + return Err(ExtractionError::Network(NetworkError::Transport(format!( + "search continuation HTTP {}", + resp.response_code() + )))); + } + let parsed: Value = serde_json::from_str(resp.response_body()) + .map_err(|e| ExtractionError::Parsing(ParsingError::JsonShape(e.to_string())))?; + Ok(parse_search_continuation(&parsed)) +} + +/// Parse a search continuation response. Items arrive under +/// `onResponseReceivedCommands[0].appendContinuationItemsAction +/// .continuationItems[]`, where each cell is an `itemSectionRenderer` +/// (its `.contents[]` are the same videoRenderer/shelfRenderer shapes as +/// page 1, walked by the shared `extract_item_into`) plus a trailing +/// continuationItemRenderer carrying the next token. +pub fn parse_search_continuation(body: &Value) -> ContinuationPage { + let Some(items) = continuation_items(body, "onResponseReceivedCommands") else { + return ContinuationPage::default(); + }; + let mut info = SearchInfo::default(); + for cell in items { + if let Some(contents) = cell + .get("itemSectionRenderer") + .and_then(|s| s.get("contents")) + .and_then(|c| c.as_array()) + { + for item in contents { + extract_item_into(item, &mut info); + } + } else { + // Defensive: some continuation responses surface bare + // renderers without the itemSectionRenderer wrapper. + // extract_item_into silently skips cells it doesn't + // recognise (including the continuationItemRenderer). + extract_item_into(cell, &mut info); + } + } + ContinuationPage { + items: info.videos, + continuation: token_from_items(items), + } +} + pub fn parse_search_response(query: &str, body: &Value) -> SearchInfo { let mut info = SearchInfo { query: query.to_string(), @@ -443,4 +514,55 @@ mod tests { assert_eq!(info.videos.len(), 1); assert_eq!(info.videos[0].name, "In shelf"); } + + #[test] + fn parses_search_continuation_items_plus_token() { + // Continuation responses land under onResponseReceivedCommands, + // wrap items in itemSectionRenderer, and tail with a + // continuationItemRenderer for the next page. + let body = json!({ + "onResponseReceivedCommands": [{ + "appendContinuationItemsAction": {"continuationItems": [ + {"itemSectionRenderer": {"contents": [ + {"videoRenderer": {"videoId":"contvideo01","title":{"simpleText":"Cont Video"}}} + ]}}, + {"continuationItemRenderer": {"continuationEndpoint": { + "continuationCommand": {"token": "SEARCH_TOKEN_2"} + }}} + ]} + }] + }); + let page = parse_search_continuation(&body); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].name, "Cont Video"); + assert_eq!(page.items[0].url, "https://www.youtube.com/watch?v=contvideo01"); + assert_eq!(page.continuation.as_deref(), Some("SEARCH_TOKEN_2")); + } + + #[test] + fn search_continuation_walks_shelf_and_marks_last_page() { + let body = json!({ + "onResponseReceivedCommands": [{ + "appendContinuationItemsAction": {"continuationItems": [ + {"itemSectionRenderer": {"contents": [ + {"shelfRenderer": {"content": {"verticalListRenderer": {"items": [ + {"videoRenderer": {"videoId":"shelfcont01","title":{"simpleText":"Shelf Cont"}}} + ]}}}} + ]}} + ]} + }] + }); + let page = parse_search_continuation(&body); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].name, "Shelf Cont"); + // No trailing continuationItemRenderer → last page. + assert_eq!(page.continuation, None); + } + + #[test] + fn search_continuation_empty_body_is_empty_page() { + let page = parse_search_continuation(&json!({})); + assert!(page.items.is_empty()); + assert_eq!(page.continuation, None); + } }