Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat!: add channel_videos_tab, channel_videos_order,

Browse source 
remove channel_shorts, channel_livestreams
fix: parsing video types and short durations
This commit is contained in:
ThetaDev 2023-05-11 14:13:54 +02:00
parent 7e5cff719a
commit 3a75ed8610
20 changed files with 444 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

36
cli/src/main.rs
View file

@ -8,7 +8,7 @@ use reqwest::{Client, ClientBuilder};
use rustypipe::{
client::RustyPipe,
model::{UrlTarget, VideoId},
param::{search_filter, StreamFilter},
param::{search_filter, ChannelVideoTab, StreamFilter},
};
use serde::Serialize;
@ -113,6 +113,7 @@ enum ChannelTab {
Videos,
Shorts,
Live,
Playlists,
Info,
}
@ -564,27 +565,16 @@ async fn main() {
print_data(&artist, format, pretty);
} else {
match tab {
ChannelTab::Videos => {
let mut channel = rp.query().channel_videos(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Shorts => {
let mut channel = rp.query().channel_shorts(&id).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
.await
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Live => {
ChannelTab::Videos | ChannelTab::Shorts | ChannelTab::Live => {
let video_tab = match tab {
ChannelTab::Videos => ChannelVideoTab::Videos,
ChannelTab::Shorts => ChannelVideoTab::Shorts,
ChannelTab::Live => ChannelVideoTab::Live,
_ => unreachable!(),
};
let mut channel =
rp.query().channel_livestreams(&id).await.unwrap();
rp.query().channel_videos_tab(&id, video_tab).await.unwrap();
channel
.content
.extend_limit(rp.query(), limit)
@ -592,6 +582,10 @@ async fn main() {
.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Playlists => {
let channel = rp.query().channel_playlists(&id).await.unwrap();
print_data(&channel, format, pretty);
}
ChannelTab::Info => {
let channel = rp.query().channel_info(&id).await.unwrap();
print_data(&channel, format, pretty);

6
codegen/src/download_testfiles.rs
View file

@ -10,7 +10,7 @@ use rustypipe::{
client::{ClientType, RustyPipe},
param::{
search_filter::{self, ItemType, SearchFilter},
Country,
ChannelVideoTab, Country,
},
report::{Report, Reporter},
};
@ -305,7 +305,7 @@ async fn channel_shorts() {
let rp = rp_testfile(&json_path);
rp.query()
.channel_shorts("UCh8gHdtzO2tXd593_bjErWg")
.channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts)
.await
.unwrap();
}
@ -318,7 +318,7 @@ async fn channel_livestreams() {
let rp = rp_testfile(&json_path);
rp.query()
.channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")
.channel_videos_tab("UC2DjFE7Xf11URZqWBigcVOQ", ChannelVideoTab::Live)
.await
.unwrap();
}

69
notes/channel_order.md Normal file
View file

@ -0,0 +1,69 @@
# Channel order
Fields:
- `2:0:string` Channel ID
- `15:0:embedded` Videos tab
- `10:0:embedded` Shorts tab
- `14:0:embedded` Livestreams tab
- `2:0:string`: targetId for YouTube's web framework (`"\n$"` + any UUID)
- `3:1:varint` Sort order (1: Latest, 2: Popular)
Popular videos
```json
{
"80226972:0:embedded": {
"2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
"3:1:base64": {
"110:0:embedded": {
"3:0:embedded": {
"15:0:embedded": {
"2:0:string": "\n$6461d7c8-0000-2040-87aa-089e0827e420",
"3:1:varint": 2
}
}
}
}
}
}
```
Popular shorts
```json
{
"80226972:0:embedded": {
"2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
"3:1:base64": {
"110:0:embedded": {
"3:0:embedded": {
"10:0:embedded": {
"2:0:string": "\n$64679ffb-0000-26b3-a1bd-582429d2c794",
"3:1:varint": 2
}
}
}
}
}
}
```
Popular streams
```json
{
"80226972:0:embedded": {
"2:0:string": "UCXuqSBlHAE6Xw-yeJA0Tunw",
"3:1:base64": {
"110:0:embedded": {
"3:0:embedded": {
"14:0:embedded": {
"2:0:string": "\n$64693069-0000-2a1e-8c7d-582429bd5ba8",
"3:1:varint": 2
}
}
}
}
}
}
```

163
src/client/channel.rs
View file

@ -1,14 +1,15 @@
use std::borrow::Cow;
use serde::Serialize;
use url::Url;
use crate::{
error::{Error, ExtractionError},
model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem},
param::Language,
model::{
paginator::{ContinuationEndpoint, Paginator},
Channel, ChannelInfo, PlaylistItem, VideoItem, YouTubeItem,
},
param::{ChannelOrder, ChannelVideoTab, Language},
serializer::MapResult,
util,
util::{self, ProtoBuilder},
};
use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
@ -18,13 +19,13 @@ use super::{response, ClientType, MapResponse, RustyPipeQuery, YTContext};
struct QChannel<'a> {
context: YTContext<'a>,
browse_id: &'a str,
params: Params,
params: ChannelTab,
#[serde(skip_serializing_if = "Option::is_none")]
query: Option<&'a str>,
}
#[derive(Debug, Serialize)]
enum Params {
enum ChannelTab {
#[serde(rename = "EgZ2aWRlb3PyBgQKAjoA")]
Videos,
#[serde(rename = "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")]
@ -39,11 +40,21 @@ enum Params {
Search,
}
impl From<ChannelVideoTab> for ChannelTab {
fn from(value: ChannelVideoTab) -> Self {
match value {
ChannelVideoTab::Videos => Self::Videos,
ChannelVideoTab::Shorts => Self::Shorts,
ChannelVideoTab::Live => Self::Live,
}
}
}
impl RustyPipeQuery {
async fn _channel_videos<S: AsRef<str>>(
&self,
channel_id: S,
params: Params,
params: ChannelTab,
query: Option<&str>,
operation: &str,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
@ -71,25 +82,51 @@ impl RustyPipeQuery {
&self,
channel_id: S,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Videos, None, "channel_videos")
self._channel_videos(channel_id, ChannelTab::Videos, None, "channel_videos")
.await
}
/// Get the short videos from a YouTube channel
pub async fn channel_shorts<S: AsRef<str>>(
/// Get a ordered list of videos from a YouTube channel
///
/// This function does not return channel metadata.
pub async fn channel_videos_order<S: AsRef<str>>(
&self,
channel_id: S,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Shorts, None, "channel_shorts")
order: ChannelOrder,
) -> Result<Paginator<VideoItem>, Error> {
self.channel_videos_tab_order(channel_id, ChannelVideoTab::Videos, order)
.await
}
/// Get the livestreams from a YouTube channel
pub async fn channel_livestreams<S: AsRef<str>>(
/// Get the specified video tab from a YouTube channel
pub async fn channel_videos_tab<S: AsRef<str>>(
&self,
channel_id: S,
tab: ChannelVideoTab,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(channel_id, Params::Live, None, "channel_livestreams")
self._channel_videos(channel_id, tab.into(), None, "channel_videos")
.await
}
/// Get a ordered list of videos from the specified tab of a YouTube channel
///
/// This function does not return channel metadata.
pub async fn channel_videos_tab_order<S: AsRef<str>>(
&self,
channel_id: S,
tab: ChannelVideoTab,
order: ChannelOrder,
) -> Result<Paginator<VideoItem>, Error> {
let visitor_data = match tab {
ChannelVideoTab::Shorts => Some(self.get_visitor_data().await?),
_ => None,
};
self.continuation(
order_ctoken(channel_id.as_ref(), tab, order),
ContinuationEndpoint::Browse,
visitor_data.as_deref(),
)
.await
}
@ -101,7 +138,7 @@ impl RustyPipeQuery {
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self._channel_videos(
channel_id,
Params::Search,
ChannelTab::Search,
Some(query.as_ref()),
"channel_search",
)
@ -118,7 +155,7 @@ impl RustyPipeQuery {
let request_body = QChannel {
context,
browse_id: channel_id,
params: Params::Playlists,
params: ChannelTab::Playlists,
query: None,
};
@ -142,7 +179,7 @@ impl RustyPipeQuery {
let request_body = QChannel {
context,
browse_id: channel_id,
params: Params::Info,
params: ChannelTab::Info,
query: None,
};
@ -451,16 +488,16 @@ fn map_channel_content(
.or(tab.tab_renderer.content.section_list_renderer)
});
let content = match channel_content {
Some(list) => list.contents,
None => {
// YouTube may show the "Featured" tab if the requested tab is empty/does not exist
if featured_tab {
let content = if featured_tab {
MapResult::default()
} else {
return Err(ExtractionError::InvalidData(Cow::Borrowed(
"could not extract content",
)));
match channel_content {
Some(list) => list.contents,
None => {
return Err(ExtractionError::InvalidData(
"could not extract content".into(),
))
}
}
};
@ -495,6 +532,47 @@ fn combine_channel_data<T>(channel_data: Channel<()>, content: T) -> Channel<T>
}
}
/// Get the continuation token to fetch channel videos in the given order
fn order_ctoken(channel_id: &str, tab: ChannelVideoTab, order: ChannelOrder) -> String {
_order_ctoken(
channel_id,
tab,
order,
&format!("\n${}", util::random_uuid()),
)
}
/// Get the continuation token to fetch channel videos in the given order
/// (fixed targetId for testing)
fn _order_ctoken(
channel_id: &str,
tab: ChannelVideoTab,
order: ChannelOrder,
target_id: &str,
) -> String {
let mut pb_tab = ProtoBuilder::new();
pb_tab.string(2, target_id);
pb_tab.varint(3, order as u64);
let mut pb_3 = ProtoBuilder::new();
pb_3.embedded(tab.order_ctoken_id(), pb_tab);
let mut pb_110 = ProtoBuilder::new();
pb_110.embedded(3, pb_3);
let mut pbi = ProtoBuilder::new();
pbi.embedded(110, pb_110);
let mut pb_80226972 = ProtoBuilder::new();
pb_80226972.string(2, channel_id);
pb_80226972.string(3, &pbi.to_base64());
let mut pb = ProtoBuilder::new();
pb.embedded(80226972, pb_80226972);
pb.to_base64()
}
#[cfg(test)]
mod tests {
use std::{fs::File, io::BufReader};
@ -505,11 +583,13 @@ mod tests {
use crate::{
client::{response, MapResponse},
model::{paginator::Paginator, Channel, ChannelInfo, PlaylistItem, VideoItem},
param::Language,
param::{ChannelOrder, ChannelVideoTab, Language},
serializer::MapResult,
util::tests::TESTFILES,
};
use super::_order_ctoken;
#[rstest]
#[case::base("videos_base", "UC2DjFE7Xf11URZqWBigcVOQ")]
#[case::music("videos_music", "UC_vmjW5e1xEHhYjY2a0kK1A")]
@ -585,4 +665,33 @@ mod tests {
);
insta::assert_ron_snapshot!("map_channel_info", map_res.c);
}
#[test]
fn order_ctoken() {
let channel_id = "UCXuqSBlHAE6Xw-yeJA0Tunw";
let videos_popular_token = _order_ctoken(
channel_id,
ChannelVideoTab::Videos,
ChannelOrder::Popular,
"\n$6461d7c8-0000-2040-87aa-089e0827e420",
);
assert_eq!(videos_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXg2S2hJbUNpUTJORFl4WkRkak9DMHdNREF3TFRJd05EQXRPRGRoWVMwd09EbGxNRGd5TjJVME1qQVlBZyUzRCUzRA%3D%3D");
let shorts_popular_token = _order_ctoken(
channel_id,
ChannelVideoTab::Shorts,
ChannelOrder::Popular,
"\n$64679ffb-0000-26b3-a1bd-582429d2c794",
);
assert_eq!(shorts_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXhTS2hJbUNpUTJORFkzT1dabVlpMHdNREF3TFRJMllqTXRZVEZpWkMwMU9ESTBNamxrTW1NM09UUVlBZyUzRCUzRA%3D%3D");
let live_popular_token = _order_ctoken(
channel_id,
ChannelVideoTab::Live,
ChannelOrder::Popular,
"\n$64693069-0000-2a1e-8c7d-582429bd5ba8",
);
assert_eq!(live_popular_token, "4qmFsgJkEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaSDhnWXVHaXh5S2hJbUNpUTJORFk1TXpBMk9TMHdNREF3TFRKaE1XVXRPR00zWkMwMU9ESTBNamxpWkRWaVlUZ1lBZyUzRCUzRA%3D%3D");
}
}

37
src/client/mod.rs
View file

@ -191,6 +191,7 @@ const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+";
const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/";
const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/";
const YOUTUBE_MUSIC_V1_URL: &str = "https://music.youtube.com/youtubei/v1/";
const YOUTUBE_HOME_URL: &str = "https://www.youtube.com/";
const YOUTUBE_MUSIC_HOME_URL: &str = "https://music.youtube.com/";
const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false";
@ -644,7 +645,7 @@ impl RustyPipe {
self.extract_client_version(
Some("https://www.youtube.com/sw.js"),
"https://www.youtube.com/results?search_query=",
"https://www.youtube.com",
YOUTUBE_HOME_URL,
None,
)
.await
@ -654,8 +655,8 @@ impl RustyPipe {
async fn extract_music_client_version(&self) -> Result<String, Error> {
self.extract_client_version(
Some("https://music.youtube.com/sw.js"),
"https://music.youtube.com",
"https://music.youtube.com",
YOUTUBE_MUSIC_HOME_URL,
YOUTUBE_MUSIC_HOME_URL,
None,
)
.await
@ -812,7 +813,7 @@ impl RustyPipe {
}
}
async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
async fn get_visitor_data(&self) -> Result<String, Error> {
log::debug!("getting YTM visitor data");
let resp = self.inner.http.get(YOUTUBE_MUSIC_HOME_URL).send().await?;
@ -903,7 +904,7 @@ impl RustyPipeQuery {
client_name: "WEB",
client_version: Cow::Owned(self.client.get_desktop_client_version().await),
platform: "DESKTOP",
original_url: Some("https://www.youtube.com/"),
original_url: Some(YOUTUBE_HOME_URL),
visitor_data,
hl,
gl,
@ -918,7 +919,7 @@ impl RustyPipeQuery {
client_name: "WEB_REMIX",
client_version: Cow::Owned(self.client.get_music_client_version().await),
platform: "DESKTOP",
original_url: Some("https://music.youtube.com/"),
original_url: Some(YOUTUBE_MUSIC_HOME_URL),
visitor_data,
hl,
gl,
@ -942,7 +943,7 @@ impl RustyPipeQuery {
request: Some(RequestYT::default()),
user: User::default(),
third_party: Some(ThirdParty {
embed_url: "https://www.youtube.com/",
embed_url: YOUTUBE_HOME_URL,
}),
},
ClientType::Android => YTContext {
@ -993,8 +994,8 @@ impl RustyPipeQuery {
.post(format!(
"{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}"
))
.header(header::ORIGIN, "https://www.youtube.com")
.header(header::REFERER, "https://www.youtube.com")
.header(header::ORIGIN, YOUTUBE_HOME_URL)
.header(header::REFERER, YOUTUBE_HOME_URL)
.header(header::COOKIE, self.client.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "1")
.header(
@ -1008,8 +1009,8 @@ impl RustyPipeQuery {
.post(format!(
"{YOUTUBE_MUSIC_V1_URL}{endpoint}?key={DESKTOP_MUSIC_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}"
))
.header(header::ORIGIN, "https://music.youtube.com")
.header(header::REFERER, "https://music.youtube.com")
.header(header::ORIGIN, YOUTUBE_MUSIC_HOME_URL)
.header(header::REFERER, YOUTUBE_MUSIC_HOME_URL)
.header(header::COOKIE, self.client.inner.consent_cookie.to_owned())
.header("X-YouTube-Client-Name", "67")
.header(
@ -1023,8 +1024,8 @@ impl RustyPipeQuery {
.post(format!(
"{YOUTUBEI_V1_URL}{endpoint}?key={DESKTOP_API_KEY}{DISABLE_PRETTY_PRINT_PARAMETER}"
))
.header(header::ORIGIN, "https://www.youtube.com")
.header(header::REFERER, "https://www.youtube.com")
.header(header::ORIGIN, YOUTUBE_HOME_URL)
.header(header::REFERER, YOUTUBE_HOME_URL)
.header("X-YouTube-Client-Name", "1")
.header("X-YouTube-Client-Version", TVHTML5_CLIENT_VERSION),
ClientType::Android => self
@ -1060,11 +1061,11 @@ impl RustyPipeQuery {
}
}
/// Get a YouTube Music visitor data cookie, which is necessary for certain requests
async fn get_ytm_visitor_data(&self) -> Result<String, Error> {
/// Get a YouTube visitor data cookie, which is necessary for certain requests
async fn get_visitor_data(&self) -> Result<String, Error> {
match &self.opts.visitor_data {
Some(vd) => Ok(vd.to_owned()),
None => self.client.get_ytm_visitor_data().await,
None => self.client.get_visitor_data().await,
}
}
@ -1304,9 +1305,9 @@ mod tests {
}
#[test]
fn t_get_ytm_visitor_data() {
fn t_get_visitor_data() {
let rp = RustyPipe::new();
let visitor_data = tokio_test::block_on(rp.get_ytm_visitor_data()).unwrap();
let visitor_data = tokio_test::block_on(rp.get_visitor_data()).unwrap();
assert!(visitor_data.ends_with("%3D"));
assert_eq!(visitor_data.len(), 32)
}

2
src/client/music_artist.rs
View file

@ -27,7 +27,7 @@ impl RustyPipeQuery {
) -> Result<MusicArtist, Error> {
let artist_id = artist_id.as_ref();
let visitor_data = match all_albums {
true => Some(self.get_ytm_visitor_data().await?),
true => Some(self.get_visitor_data().await?),
false => None,
};

2
src/client/music_details.rs
View file

@ -109,7 +109,7 @@ impl RustyPipeQuery {
radio_id: S,
) -> Result<Paginator<TrackItem>, Error> {
let radio_id = radio_id.as_ref();
let visitor_data = self.get_ytm_visitor_data().await?;
let visitor_data = self.get_visitor_data().await?;
let context = self
.get_context(ClientType::DesktopMusic, true, Some(&visitor_data))
.await;

6
src/client/pagination.rs
View file

@ -102,8 +102,12 @@ impl MapResponse<Paginator<YouTubeItem>> for response::Continuation {
.and_then(|actions| {
actions
.into_iter()
.next()
.map(|action| action.append_continuation_items_action.continuation_items)
.reduce(|mut acc, mut items| {
acc.c.append(&mut items.c);
acc.warnings.append(&mut items.warnings);
acc
})
})
.or_else(|| {
self.continuation_contents

14
src/client/player.rs
View file

@ -492,8 +492,6 @@ fn map_audio_stream(
deobf: &Deobfuscator,
last_nsig: &mut [String; 2],
) -> MapResult<Option<AudioStream>> {
static LANG_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"^([a-z]{2,3})\."#).unwrap());
let (mtype, codecs) = match parse_mime(&f.mime_type) {
Some(x) => x,
None => {
@ -535,18 +533,12 @@ fn map_audio_stream(
loudness_db: f.loudness_db,
throttled,
track: match f.audio_track {
Some(t) => {
let lang = LANG_PATTERN
.captures(&t.id)
.map(|m| m.get(1).unwrap().as_str().to_owned());
Some(AudioTrack {
Some(t) => Some(AudioTrack {
lang: t.id.split('.').next().map(str::to_owned),
id: t.id,
lang,
lang_name: t.display_name,
is_default: t.audio_is_default,
})
}
}),
None => None,
},
}),

1
src/client/response/mod.rs
View file

@ -219,6 +219,7 @@ pub(crate) struct Continuation {
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct ContinuationActionWrap {
#[serde(alias = "reloadContinuationItemsCommand")]
pub append_continuation_items_action: ContinuationAction,
}

25
src/client/response/video_item.rs
View file

@ -1,5 +1,3 @@
use once_cell::sync::Lazy;
use regex::Regex;
use serde::Deserialize;
use serde_with::{
json::JsonString, rust::deserialize_ignore_any, serde_as, DefaultOnError, VecSkipError,
@ -430,11 +428,17 @@ impl<T> YouTubeListMapper<T> {
}
fn map_video(&mut self, video: VideoRenderer) -> VideoItem {
let mut tn_overlays = video.thumbnail_overlays;
let is_live = video.thumbnail_overlays.is_live() || video.badges.is_live();
let is_short = video.thumbnail_overlays.is_short();
let length_text = video.length_text.or_else(|| {
tn_overlays
.try_swap_remove(0)
.map(|overlay| overlay.thumbnail_overlay_time_status_renderer.text)
video
.thumbnail_overlays
.into_iter()
.find(|ol| {
ol.thumbnail_overlay_time_status_renderer.style == TimeOverlayStyle::Default
})
.map(|ol| ol.thumbnail_overlay_time_status_renderer.text)
});
VideoItem {
@ -472,8 +476,8 @@ impl<T> YouTubeListMapper<T> {
view_count: video
.view_count_text
.map(|txt| util::parse_numeric(&txt).unwrap_or_default()),
is_live: tn_overlays.is_live() || video.badges.is_live(),
is_short: tn_overlays.is_short(),
is_live,
is_short,
is_upcoming: video.upcoming_event_data.is_some(),
short_description: video
.detailed_metadata_snippets
@ -483,9 +487,6 @@ impl<T> YouTubeListMapper<T> {
}
fn map_short_video(&mut self, video: ReelItemRenderer, lang: Language) -> VideoItem {
static ACCESSIBILITY_SEP_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(" [-\u{2013}] ").unwrap());
let pub_date_txt = video.navigation_endpoint.map(|n| {
n.reel_watch_endpoint
.overlay
@ -499,7 +500,7 @@ impl<T> YouTubeListMapper<T> {
id: video.video_id,
name: video.headline,
length: video.accessibility.and_then(|acc| {
ACCESSIBILITY_SEP_REGEX.split(&acc).nth(1).and_then(|s| {
acc.rsplit(" - ").nth(1).and_then(|s| {
timeago::parse_video_duration_or_warn(self.lang, s, &mut self.warnings)
})
}),

8
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_live.snap
View file

@ -168,7 +168,7 @@ Channel(
publish_date: "[date]",
publish_date_txt: None,
view_count: Some(94),
is_live: false,
is_live: true,
is_short: false,
is_upcoming: false,
short_description: None,
@ -209,7 +209,7 @@ Channel(
publish_date: "[date]",
publish_date_txt: None,
view_count: Some(381),
is_live: false,
is_live: true,
is_short: false,
is_upcoming: false,
short_description: None,
@ -414,7 +414,7 @@ Channel(
publish_date: "[date]",
publish_date_txt: None,
view_count: Some(2043),
is_live: false,
is_live: true,
is_short: false,
is_upcoming: false,
short_description: None,
@ -783,7 +783,7 @@ Channel(
publish_date: "[date]",
publish_date_txt: None,
view_count: Some(4030),
is_live: false,
is_live: true,
is_short: false,
is_upcoming: false,
short_description: None,

48
src/client/snapshots/rustypipe__client__channel__tests__map_channel_videos_shorts.snap
View file

@ -141,7 +141,7 @@ Channel(
publish_date_txt: Some("1 day ago"),
view_count: Some(443549),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -167,7 +167,7 @@ Channel(
publish_date_txt: Some("2 days ago"),
view_count: Some(1154962),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -234,7 +234,7 @@ Channel(
publish_date_txt: Some("6 days ago"),
view_count: Some(1388173),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -260,7 +260,7 @@ Channel(
publish_date_txt: Some("7 days ago"),
view_count: Some(1738301),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -286,7 +286,7 @@ Channel(
publish_date_txt: Some("9 days ago"),
view_count: Some(1316594),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -353,7 +353,7 @@ Channel(
publish_date_txt: Some("11 days ago"),
view_count: Some(1412213),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -379,7 +379,7 @@ Channel(
publish_date_txt: Some("13 days ago"),
view_count: Some(1513305),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -405,7 +405,7 @@ Channel(
publish_date_txt: Some("2 weeks ago"),
view_count: Some(8936223),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -472,7 +472,7 @@ Channel(
publish_date_txt: Some("2 weeks ago"),
view_count: Some(2769717),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -539,7 +539,7 @@ Channel(
publish_date_txt: Some("3 weeks ago"),
view_count: Some(572107),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -565,7 +565,7 @@ Channel(
publish_date_txt: Some("3 weeks ago"),
view_count: Some(1707132),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -591,7 +591,7 @@ Channel(
publish_date_txt: Some("3 weeks ago"),
view_count: Some(933094),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -617,7 +617,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(5985184),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -643,7 +643,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(14741387),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -669,7 +669,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(2511322),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -695,7 +695,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(2364408),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -762,7 +762,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(1947627),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -788,7 +788,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(4763839),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -814,7 +814,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(1915695),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -840,7 +840,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(7268944),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -866,7 +866,7 @@ Channel(
publish_date_txt: Some("1 month ago"),
view_count: Some(2539103),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -892,7 +892,7 @@ Channel(
publish_date_txt: Some("2 months ago"),
view_count: Some(5545680),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -918,7 +918,7 @@ Channel(
publish_date_txt: Some("2 months ago"),
view_count: Some(2202314),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),
@ -985,7 +985,7 @@ Channel(
publish_date_txt: Some("2 months ago"),
view_count: Some(6443699),
is_live: false,
is_short: false,
is_short: true,
is_upcoming: false,
short_description: None,
),

2
src/model/mod.rs
View file

@ -850,7 +850,7 @@ pub struct VideoItem {
pub publish_date: Option<OffsetDateTime>,
/// Textual video publish date (e.g. `11 months ago`, depends on language)
///
/// Is [`None`] for livestreams.
/// Is [`None`] for livestreams and upcoming videos.
pub publish_date_txt: Option<String>,
/// View count
///

2
src/model/paginator.rs
View file

@ -34,7 +34,7 @@ pub struct Paginator<T> {
#[serde(skip_serializing_if = "Option::is_none")]
pub visitor_data: Option<String>,
/// YouTube API endpoint to fetch continuations from
pub(crate) endpoint: ContinuationEndpoint,
pub endpoint: ContinuationEndpoint,
}
impl<T> Default for Paginator<T> {

31
src/param/mod.rs
View file

@ -7,3 +7,34 @@ pub mod search_filter;
pub use locale::{Country, Language};
pub use stream_filter::StreamFilter;
/// Channel video tab
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChannelVideoTab {
/// Regular videos
Videos,
/// Short videos
Shorts,
/// Livestreams
Live,
}
/// Sort order for channel videos
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChannelOrder {
/// Order videos with the latest upload date first (default)
Latest = 1,
/// Order videos with the highest number of views first
Popular = 2,
}
impl ChannelVideoTab {
/// Get the tab ID used to create ordered continuation tokens
pub(crate) const fn order_ctoken_id(&self) -> u32 {
match self {
ChannelVideoTab::Videos => 15,
ChannelVideoTab::Shorts => 10,
ChannelVideoTab::Live => 14,
}
}
}

5
src/param/search_filter.rs
View file

@ -2,7 +2,7 @@
use std::collections::BTreeSet;
use crate::util::{self, ProtoBuilder};
use crate::util::ProtoBuilder;
/// YouTube search filter
///
@ -200,8 +200,7 @@ impl SearchFilter {
pb.embedded(8, extras)
}
let b64 = util::b64_encode(pb.bytes);
urlencoding::encode(&b64).to_string()
pb.to_base64()
}
}

12
src/util/mod.rs
View file

@ -83,6 +83,18 @@ pub fn generate_content_playback_nonce() -> String {
random_string(CONTENT_PLAYBACK_NONCE_ALPHABET, 16)
}
pub fn random_uuid() -> String {
let mut rng = rand::thread_rng();
format!(
"{:08x}-{:04x}-{:04x}-{:04x}-{:012x}",
rng.gen::<u32>(),
rng.gen::<u16>(),
rng.gen::<u16>(),
rng.gen::<u16>(),
rng.gen::<u64>() & 0xffffffffffff,
)
}
/// Split an URL into its base string and parameter map
///
/// Example:

18
src/util/protobuf.rs
View file

@ -45,6 +45,13 @@ impl ProtoBuilder {
self._varint(val);
}
/// Write a string field
pub fn string(&mut self, field: u32, string: &str) {
self._field(field, 2);
self._varint(string.len() as u64);
self.bytes.extend_from_slice(string.as_bytes());
}
/// Write an embedded message
///
/// Requires passing another [`ProtoBuilder`] with the embedded message.
@ -53,6 +60,12 @@ impl ProtoBuilder {
self._varint(pb.bytes.len() as u64);
self.bytes.append(&mut pb.bytes);
}
/// Base64 + urlencode the protobuf data
pub fn to_base64(&self) -> String {
let b64 = super::b64_encode(&self.bytes);
urlencoding::encode(&b64).to_string()
}
}
fn parse_varint<P: Iterator<Item = u8>>(pb: &mut P) -> Option<u64> {
@ -124,11 +137,6 @@ mod tests {
use super::*;
// #[test]
// fn t_parse_varint() {
// }
#[test]
fn t_parse_proto() {
let p = "GhhVQzl2cnZOU0wzeGNXR1NrVjg2UkVCU2c%3D";

97
tests/youtube.rs
View file

@ -4,7 +4,7 @@ use std::str::FromStr;
use rstest::{fixture, rstest};
use rustypipe::model::paginator::ContinuationEndpoint;
use rustypipe::param::Language;
use rustypipe::param::{ChannelOrder, ChannelVideoTab, Language};
use rustypipe::validate;
use time::macros::date;
use time::OffsetDateTime;
@ -261,15 +261,10 @@ fn get_player(
let langs = player_data
.audio_streams
.iter()
.filter_map(|stream| {
stream
.track
.as_ref()
.map(|t| t.lang.as_ref().unwrap().to_owned())
})
.filter_map(|stream| stream.track.as_ref().map(|t| t.lang.as_deref().unwrap()))
.collect::<HashSet<_>>();
for l in ["en", "es", "fr", "pt", "ru"] {
for l in ["en-US", "es", "fr", "pt", "ru"] {
assert!(langs.contains(l), "missing lang: {l}");
}
}
@ -779,8 +774,11 @@ fn channel_videos(rp: RustyPipe) {
#[rstest]
fn channel_shorts(rp: RustyPipe) {
let channel =
tokio_test::block_on(rp.query().channel_shorts("UCh8gHdtzO2tXd593_bjErWg")).unwrap();
let channel = tokio_test::block_on(
rp.query()
.channel_videos_tab("UCh8gHdtzO2tXd593_bjErWg", ChannelVideoTab::Shorts),
)
.unwrap();
// dbg!(&channel);
assert_eq!(channel.id, "UCh8gHdtzO2tXd593_bjErWg");
@ -809,8 +807,11 @@ fn channel_shorts(rp: RustyPipe) {
#[rstest]
fn channel_livestreams(rp: RustyPipe) {
let channel =
tokio_test::block_on(rp.query().channel_livestreams("UC2DjFE7Xf11URZqWBigcVOQ")).unwrap();
let channel = tokio_test::block_on(
rp.query()
.channel_videos_tab("UC2DjFE7Xf11URZqWBigcVOQ", ChannelVideoTab::Live),
)
.unwrap();
// dbg!(&channel);
assert_channel_eevblog(&channel);
@ -955,6 +956,63 @@ fn channel_more(
assert_channel(&channel_info, id, name, unlocalized || name_unlocalized);
}
#[rstest]
#[case::videos("UCcdwLMPsaU2ezNSJU1nFoBQ", ChannelVideoTab::Videos, "XqZsoesa55w")]
#[case::shorts("UCcdwLMPsaU2ezNSJU1nFoBQ", ChannelVideoTab::Shorts, "k91vRvXGwHs")]
#[case::live("UCvqRdlKsE5Q8mf8YXbdIJLw", ChannelVideoTab::Live, "ojes5ULOqhc")]
fn channel_order(
#[case] id: &str,
#[case] tab: ChannelVideoTab,
#[case] most_popular: &str,
rp: RustyPipe,
) {
let latest = tokio_test::block_on(rp.query().channel_videos_tab_order(
id,
tab,
ChannelOrder::Latest,
))
.unwrap();
// Upload dates should be in descending order
if tab != ChannelVideoTab::Shorts {
let mut latest_items = latest.items.iter().peekable();
while let (Some(v), Some(next_v)) = (latest_items.next(), latest_items.peek()) {
if !v.is_upcoming && !v.is_live && !next_v.is_upcoming && !next_v.is_live {
assert_gte(
v.publish_date.unwrap(),
next_v.publish_date.unwrap(),
"latest video date",
);
}
}
}
assert_next(latest, rp.query(), 15, 2);
let popular = tokio_test::block_on(rp.query().channel_videos_tab_order(
id,
tab,
ChannelOrder::Popular,
))
.unwrap();
// Most popular video should be in top 5
assert!(
popular.items.iter().take(5).any(|v| v.id == most_popular),
"most popular video {most_popular} not found"
);
// View counts should be in descending order
if tab != ChannelVideoTab::Shorts {
let mut popular_items = popular.items.iter().peekable();
while let (Some(v), Some(next_v)) = (popular_items.next(), popular_items.peek()) {
assert_gte(
v.view_count.unwrap(),
next_v.view_count.unwrap(),
"most popular view count",
);
}
}
assert_next(popular, rp.query(), 15, 2);
}
#[rstest]
#[case::not_exist("UCOpNcN46UbXVtpKMrmU4Abx")]
#[case::gaming("UCOpNcN46UbXVtpKMrmU4Abg")]
@ -972,6 +1030,19 @@ fn channel_not_found(#[case] id: &str, rp: RustyPipe) {
);
}
#[rstest]
#[case::shorts(ChannelVideoTab::Shorts)]
#[case::live(ChannelVideoTab::Live)]
fn channel_tab_not_found(#[case] tab: ChannelVideoTab, rp: RustyPipe) {
let channel = tokio_test::block_on(
rp.query()
.channel_videos_tab("UCGiJh0NZ52wRhYKYnuZI08Q", tab),
)
.unwrap();
assert!(channel.content.is_empty(), "got: {:?}", channel.content);
}
//#CHANNEL_RSS
#[cfg(feature = "rss")]
@ -2240,7 +2311,7 @@ fn assert_approx(left: f64, right: f64) {
/// Assert that number A is greater than or equal to number B
fn assert_gte<T: PartialOrd + Display>(a: T, b: T, msg: &str) {
assert!(a >= b, "expected {b} {msg}, got {a}");
assert!(a >= b, "expected >= {b} {msg}, got {a}");
}
/// Assert that the paginator produces at least n pages