Sulkta-Coop/rustypipe
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: support AB3 (channel handles in search results)

Browse source
This commit is contained in:
ThetaDev 2022-11-22 01:30:30 +01:00
parent 73fa0295bf
commit aaffc6404d
14 changed files with 5855 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

57
codegen/src/abtest.rs
View file

@ -32,40 +32,72 @@ struct QVideo<'a> {
racy_check_ok: bool,
}
pub async fn run_test(ab: ABTest, n: usize, concurrency: usize) -> usize {
pub async fn run_test(
ab: ABTest,
n: usize,
concurrency: usize,
) -> (usize, Option<String>, Option<String>) {
eprintln!("🧪 A/B test #{}: {:?}", ab as u16, ab);
let rp = RustyPipe::new();
let pb = ProgressBar::new(n as u64);
let http = reqwest::Client::default();
pb.set_style(
ProgressStyle::with_template(
"{msg} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}",
)
.unwrap(),
);
// let mut count = 0;
let results = stream::iter(0..n)
.map(|_| {
let rp = rp.clone();
let pb = pb.clone();
let http = http.clone();
async move {
let visitor_data = get_visitor_data(&http).await;
let is_present = match ab {
ABTest::AttributedTextDescription => attributed_text_description(&rp).await,
ABTest::ThreeTabChannelLayout => three_tab_channel_layout(&rp).await,
ABTest::AttributedTextDescription => {
attributed_text_description(&rp, &visitor_data).await
}
ABTest::ThreeTabChannelLayout => {
three_tab_channel_layout(&rp, &visitor_data).await
}
}
.unwrap();
pb.inc(1);
is_present
(is_present, visitor_data)
}
})
.buffer_unordered(concurrency)
.collect::<Vec<_>>()
.await;
let count = results.iter().filter(|x| **x).count();
let count = results.iter().filter(|(p, _)| *p).count();
let vd_present = results
.iter()
.find_map(|(p, vd)| if *p { Some(vd.to_owned()) } else { None });
let vd_absent = results
.iter()
.find_map(|(p, vd)| if !*p { Some(vd.to_owned()) } else { None });
count
(count, vd_present, vd_absent)
}
async fn get_visitor_data(http: &reqwest::Client) -> String {
let resp = http.get("https://www.youtube.com").send().await.unwrap();
resp.headers()
.get_all(reqwest::header::SET_COOKIE)
.iter()
.find_map(|c| {
if let Ok(cookie) = c.to_str() {
if let Some(after) = cookie.strip_prefix("__Secure-YEC=") {
return after.split_once(';').map(|s| s.0.to_owned());
}
}
None
})
.unwrap()
}
pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec<ABTestRes> {
@ -73,7 +105,7 @@ pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec<ABTestRes> {
for id in 1..=N_TESTS {
let ab = ABTest::try_from(id).unwrap();
let occurrences = run_test(ab, n, concurrency).await;
let (occurrences, _, _) = run_test(ab, n, concurrency).await;
results.push(ABTestRes {
id,
name: ab,
@ -84,9 +116,11 @@ pub async fn run_all_tests(n: usize, concurrency: usize) -> Vec<ABTestRes> {
results
}
pub async fn attributed_text_description(rp: &RustyPipe) -> Result<bool> {
pub async fn attributed_text_description(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
let query = rp.query();
let context = query.get_context(ClientType::Desktop, true, None).await;
let context = query
.get_context(ClientType::Desktop, true, Some(visitor_data))
.await;
let q = QVideo {
context,
video_id: "ZeerrnuLi5E",
@ -102,9 +136,10 @@ pub async fn attributed_text_description(rp: &RustyPipe) -> Result<bool> {
Ok(response_txt.contains("\"attributedDescription\""))
}
pub async fn three_tab_channel_layout(rp: &RustyPipe) -> Result<bool> {
pub async fn three_tab_channel_layout(rp: &RustyPipe, visitor_data: &str) -> Result<bool> {
let channel = rp
.query()
.visitor_data(visitor_data)
.channel_videos("UCR-DXc1voovS8nhAvccRZhg")
.await
.unwrap();

12
codegen/src/main.rs
View file

@ -73,8 +73,16 @@ async fn main() {
match id {
Some(id) => {
let ab = abtest::ABTest::try_from(id).expect("invalid A/B test id");
let res = abtest::run_test(ab, n, cli.concurrency).await;
eprintln!("{} occurences", res);
let (occurrences, vd_present, vd_absent) =
abtest::run_test(ab, n, cli.concurrency).await;
eprintln!(
"{}/{} occurences ({:.1}%)",
occurrences,
n,
occurrences as f32 / n as f32 * 100.0
);
eprintln!("visitor_data (present): {:?}", vd_present);
eprintln!("visitor_data (absent): {:?}", vd_absent);
}
None => {
let res = abtest::run_all_tests(n, cli.concurrency).await;

268
notes/AB_Tests.md Normal file
View file

@ -0,0 +1,268 @@
# A/B-Tests
When YouTube introduces a new feature, it does so gradually. When a user creates a new
session, YouTube decided randomly which new features should be enabled.
YouTube sessions are identified by the visitor data cookie. This cookie is sent with every
API request using the `context.client.visitor_data` JSON parameter. It is also returned in the
`responseContext.visitorData` response parameter and stored as the `__SECURE-YEC` cookie.
By sending the same visitor data cookie, A/B tests can be reproduced, which is important for testing
alternative YouTube clients.
This page lists all A/B tests that were encountered while maintaining the RustyPipe client.
**Impact rating:**
The impact ratings shows how much effort it takes to adapt alternative YouTube clients to the
new feature.
- 🟢 **Low** Minor incompatibility (e.g. parameter name change)
- 🟡 **Medium** Extensive changes to the response data model OR removal of parameters
- 🔴 **High** Changes to the functionality of YouTube that will require API changes
for alternative clients
If you want to check how often these A/B tests occur, you can use the `codegen` tool with the
following command: `rustypipe-codegen ab-test <id>`.
## [1] Attributed text description
- **Encountered on:** 24.09.2022
- **Impact:** 🟡 Medium
- **Endpoint:** next (video details)
![A/B test 1 screenshot](./_img/ab_1.png)
YouTube shows internal links (channels, videos, playlists) in the video description
as buttons with the YouTube icon. To accomplish this, they completely changed the underlying
data model.
The new format uses a string with the entire plaintext content along with a list of `"commandRuns"`
which include the link data and the position of the links within the text.
Note that the position and length parameter refer to the number of UTF-16 characters. If
you are implementing this in a language which does not use UTF-16 as its internal string
representation, you have to iterate over the unicode codepoints and keep track of the UTF-16
index seperately.
**OLD**
```json
{
"videoSecondaryInfoRenderer": {
"description": {
"runs": [
{
"text": "🎧Listen and download aespa's debut single \"Black Mamba\": "
},
{
"navigationEndpoint": {
"commandMetadata": {
"webCommandMetadata": {
"rootVe": 83769,
"url": "https://www.youtube.com/redirect?...",
"webPageType": "WEB_PAGE_TYPE_UNKNOWN"
}
},
"urlEndpoint": {
"nofollow": true,
"target": "TARGET_NEW_WINDOW",
"url": "https://www.youtube.com/redirect?..."
}
},
"text": "https://smarturl.it/aespa_BlackMamba"
}
]
}
}
}
```
**NEW**
```json
{
"videoSecondaryInfoRenderer": {
"attributedDescription": {
"content": "🎧Listen and download aespa's debut single \"Black Mamba\": https://smarturl.it/aespa_BlackMamba\n🐍The Debut Stage...",
"commandRuns": [
{
"startIndex": 58,
"length": 36,
"onTap": {
"innertubeCommand": {
"commandMetadata": {
"webCommandMetadata": {
"url": "https://www.youtube.com/redirect?...",
"webPageType": "WEB_PAGE_TYPE_UNKNOWN",
"rootVe": 83769
}
},
"urlEndpoint": {
"url": "https://www.youtube.com/redirect?...",
"target": "TARGET_NEW_WINDOW",
"nofollow": true
}
}
}
}
]
}
}
}
```
## [2] 3-tab channel layout
- **Announced:** 15.09.2022, https://www.youtube.com/watch?v=czIyqEC4V-s
- **Encountered on:** 11.10.2022
- **Impact:** 🔴 High
- **Endpoint:** browse (channel videos)
![A/B test 2 screenshot](./_img/ab_2.webp)
YouTube changed their channel page layout, putting livestreams and short videos into
separate tabs.
Fetching the videos page now only returns a subset of a channel's videos. To get all videos
from a channel, you would have to run up to 3 queries.
Even though it has its disadvantages, the RSS feed is now probably the best way for keeping
track of a channel's new uploads.
Additionally the channel tab response model was slightly changed, now using a `"RichGridRenderer"`.
Short videos also have their own data models (`"reelItemRenderer"`).
**RichGrid**
```json
{
"tabRenderer": {
"content": {
"richGridRenderer": {
"contents": [
{
"richItemRenderer": {
"content": {
"videoRenderer": {}
}
}
}
]
}
}
}
}
```
**Short video**
```json
{
"reelItemRenderer": {
"accessibility": {
"accessibilityData": {
"label": "being smart was my personality trait - 56 seconds - play video"
}
},
"headline": {
"simpleText": "being smart was my personality trait"
},
"navigationEndpoint": {
"clickTrackingParams": "CLcCEIf2BBgAIhMImuP85t-D-wIVd-sRCB2r6gl7",
"commandMetadata": {
"webCommandMetadata": {
"rootVe": 37414,
"url": "/shorts/glyJWxp7a5g",
"webPageType": "WEB_PAGE_TYPE_SHORTS"
}
},
"reelWatchEndpoint": {
"overlay": {
"reelPlayerOverlayRenderer": {
"reelPlayerHeaderSupportedRenderers": {
"reelPlayerHeaderRenderer": {
"timestampText": {
"simpleText": "2 days ago"
}
}
}
}
}
}
},
"thumbnail": {
"thumbnails": [
{
"height": 720,
"url": "https://i.ytimg.com/vi/glyJWxp7a5g/hq720_2.jpg?sqp=-oaymwEdCJUDENAFSFXyq4qpAw8IARUAAIhCcAHAAQbQAQE=&rs=AOn4CLCUzo9AlrNh4n4cZfTOB8_Gf5aAkw",
"width": 405
}
]
},
"videoId": "glyJWxp7a5g",
"viewCountText": {
"simpleText": "593K views"
}
}
}
```
## [3] Channel handles in search results
- **Encountered on:** 20.11.2022
- **Impact:** 🟡 Medium
- **Endpoint:** search
![A/B test 3 screenshot](./_img/ab_3.png)
Instead of subscriber count / video count, a channel item from the search result now
displays the channel handle and the subscriber count.
The implementation looks pretty quick and dirty, as they did not even bother to rename
their response parameters. So this might change again in the future.
Note that channels without handles still use the old data model, even on the same page.
**OLD**
```json
{
"subscriberCountText": {
"accessibility": {
"accessibilityData": {
"label": "2.92 million subscribers"
}
},
"simpleText": "2.92M subscribers"
},
"videoCountText": {
"runs": [
{
"text": "219"
},
{
"text": " videos"
}
]
}
}
```
**NEW**
```json
{
"videoCountText": {
"accessibility": {
"accessibilityData": {
"label": "4.03 million subscribers"
}
},
"simpleText": "4.03M subscribers"
},
"subscriberCountText": {
"simpleText": "@MusicTravelLove"
}
}
```

BIN
notes/_img/ab_1.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 64 KiB

BIN
notes/_img/ab_2.webp Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 467 KiB

BIN
notes/_img/ab_3.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 34 KiB

76
src/client/response/video_item.rs
View file

@ -399,7 +399,7 @@ impl<T> YouTubeListMapper<T> {
}
}
fn map_video(&self, video: VideoRenderer) -> VideoItem {
fn map_video(&mut self, video: VideoRenderer) -> VideoItem {
let mut tn_overlays = video.thumbnail_overlays;
let length_text = video.length_text.or_else(|| {
tn_overlays
@ -434,10 +434,9 @@ impl<T> YouTubeListMapper<T> {
.as_ref()
.and_then(|upc| OffsetDateTime::from_unix_timestamp(upc.start_time).ok())
.or_else(|| {
video
.published_time_text
.as_ref()
.and_then(|txt| timeago::parse_timeago_to_dt(self.lang, txt))
video.published_time_text.as_ref().and_then(|txt| {
timeago::parse_timeago_dt_or_warn(self.lang, txt, &mut self.warnings)
})
}),
publish_date_txt: video.published_time_text,
view_count: video
@ -453,7 +452,7 @@ impl<T> YouTubeListMapper<T> {
}
}
fn map_short_video(&self, video: ReelItemRenderer) -> VideoItem {
fn map_short_video(&mut self, video: ReelItemRenderer) -> VideoItem {
static ACCESSIBILITY_SEP_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(" [-\u{2013}] (.+) [-\u{2013}] ").unwrap());
@ -476,16 +475,20 @@ impl<T> YouTubeListMapper<T> {
.flatten()
.and_then(|cap| {
cap.get(1).and_then(|c| {
timeago::parse_timeago(self.lang, c.as_str())
.map(|ta| Duration::from(ta).whole_seconds() as u32)
timeago::parse_timeago_or_warn(
self.lang,
c.as_str(),
&mut self.warnings,
)
.map(|ta| Duration::from(ta).whole_seconds() as u32)
})
})
}),
thumbnail: video.thumbnail.into(),
channel: self.channel.clone(),
publish_date: pub_date_txt
.as_ref()
.and_then(|txt| timeago::parse_timeago_to_dt(self.lang, txt)),
publish_date: pub_date_txt.as_ref().and_then(|txt| {
timeago::parse_timeago_dt_or_warn(self.lang, txt, &mut self.warnings)
}),
publish_date_txt: pub_date_txt,
view_count: video
.view_count_text
@ -526,19 +529,27 @@ impl<T> YouTubeListMapper<T> {
}
}
fn map_channel(channel: ChannelRenderer) -> ChannelItem {
fn map_channel(&mut self, channel: ChannelRenderer) -> ChannelItem {
// channel handle instead of subscriber count (A/B test 3)
let (sc_txt, vc_text) = match channel
.subscriber_count_text
.as_ref()
.map(|txt| txt.starts_with('@'))
.unwrap_or_default()
{
true => (channel.video_count_text, None),
false => (channel.subscriber_count_text, channel.video_count_text),
};
ChannelItem {
id: channel.channel_id,
name: channel.title,
avatar: channel.thumbnail.into(),
verification: channel.owner_badges.into(),
subscriber_count: channel
.subscriber_count_text
.and_then(|txt| util::parse_numeric(&txt).ok()),
video_count: channel
.video_count_text
.and_then(|txt| util::parse_numeric(&txt).ok())
.unwrap_or_default(),
subscriber_count: sc_txt
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
video_count: vc_text
.and_then(|txt| util::parse_numeric_or_warn(&txt, &mut self.warnings)),
short_description: channel.description_snippet,
}
}
@ -548,18 +559,20 @@ impl YouTubeListMapper<YouTubeItem> {
fn map_item(&mut self, item: YouTubeListItem) {
match item {
YouTubeListItem::VideoRenderer(video) => {
self.items.push(YouTubeItem::Video(self.map_video(video)));
let mapped = YouTubeItem::Video(self.map_video(video));
self.items.push(mapped);
}
YouTubeListItem::ReelItemRenderer(video) => {
self.items
.push(YouTubeItem::Video(self.map_short_video(video)));
let mapped = self.map_short_video(video);
self.items.push(YouTubeItem::Video(mapped));
}
YouTubeListItem::PlaylistRenderer(playlist) => {
let mapped = YouTubeItem::Playlist(self.map_playlist(playlist));
self.items.push(mapped);
}
YouTubeListItem::PlaylistRenderer(playlist) => self
.items
.push(YouTubeItem::Playlist(self.map_playlist(playlist))),
YouTubeListItem::ChannelRenderer(channel) => {
self.items
.push(YouTubeItem::Channel(Self::map_channel(channel)));
let mapped = YouTubeItem::Channel(self.map_channel(channel));
self.items.push(mapped);
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
@ -588,10 +601,12 @@ impl YouTubeListMapper<VideoItem> {
fn map_item(&mut self, item: YouTubeListItem) {
match item {
YouTubeListItem::VideoRenderer(video) => {
self.items.push(self.map_video(video));
let mapped = self.map_video(video);
self.items.push(mapped);
}
YouTubeListItem::ReelItemRenderer(video) => {
self.items.push(self.map_short_video(video));
let mapped = self.map_short_video(video);
self.items.push(mapped);
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,
@ -620,7 +635,8 @@ impl YouTubeListMapper<PlaylistItem> {
fn map_item(&mut self, item: YouTubeListItem) {
match item {
YouTubeListItem::PlaylistRenderer(playlist) => {
self.items.push(self.map_playlist(playlist))
let mapped = self.map_playlist(playlist);
self.items.push(mapped)
}
YouTubeListItem::ContinuationItemRenderer {
continuation_endpoint,

3
src/client/search.rs
View file

@ -138,7 +138,8 @@ mod tests {
#[rstest]
#[case::default("default")]
#[case::playlists("playlists")]
#[case::playlists("empty")]
#[case::empty("empty")]
#[case::ab3_channel_handles("20221121_AB3_channel_handles")]
fn t_map_search(#[case] name: &str) {
let filename = format!("testfiles/search/{}.json", name);
let json_path = Path::new(&filename);

415
src/client/snapshots/rustypipe__client__search__tests__map_search_20221121_AB3_channel_handles.snap Normal file
View file

@ -0,0 +1,415 @@
---
source: src/client/search.rs
expression: map_res.c
---
SearchResult(
items: Paginator(
count: Some(476743),
items: [
Channel(ChannelItem(
id: "UCMwePVHRpDdfeUcwtDZu2Dw",
name: "Monstafluff Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/ytc/AMLnZu9YhTzdAoL6P4PYq51PCF076ITDrgLitxSDPqv6sw=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/ytc/AMLnZu9YhTzdAoL6P4PYq51PCF076ITDrgLitxSDPqv6sw=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(582),
video_count: None,
short_description: "Music Submissions: https://monstafluff.edmdistrict.com/",
)),
Channel(ChannelItem(
id: "UCLxAS02eWvfZK4icRNzWD_g",
name: "Music Travel Love",
avatar: [
Thumbnail(
url: "https://yt3.ggpht.com/ytc/AMLnZu9njNDLU_VtFjfGUaTArBp4AJFhJIxb_CxP7knf3A=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "https://yt3.ggpht.com/ytc/AMLnZu9njNDLU_VtFjfGUaTArBp4AJFhJIxb_CxP7knf3A=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Artist,
subscriber_count: Some(403),
video_count: None,
short_description: "Welcome to the official Music Travel Love YouTube channel! We travel the world making music, friends, videos and memories!",
)),
Channel(ChannelItem(
id: "UCxKxjNPyL9UO5LRWHzp5JxA",
name: "Black&White Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/FDjW2-Cb6tFbtNv02D1UX4XtvP7P3eEWB93hGimeP4pb2TadVhAgxSVMZLZDp5NiBWGLT5eprA=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/FDjW2-Cb6tFbtNv02D1UX4XtvP7P3eEWB93hGimeP4pb2TadVhAgxSVMZLZDp5NiBWGLT5eprA=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(167),
video_count: None,
short_description: "MUSIC IN HARMONY WITH YOUR LIFE!!! If any producer, label, artist or photographer has an issue with any of the music or\u{a0}...",
)),
Channel(ChannelItem(
id: "UCGIygiYkKxn7g7fFNFdXskg",
name: "HAEVN MUSIC",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/EYlGIfqhvwtfkCyi5vpqfY_kDHr6L3OeCmkudNiAyhvz6UCnTZQOQaM-8PelFDGofdIqeF7Mb4E=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/EYlGIfqhvwtfkCyi5vpqfY_kDHr6L3OeCmkudNiAyhvz6UCnTZQOQaM-8PelFDGofdIqeF7Mb4E=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Artist,
subscriber_count: Some(411),
video_count: None,
short_description: "The official YouTube channel of HAEVN Music. Receiving a piano from his grandfather had a great impact on Jorrit\'s life.",
)),
Channel(ChannelItem(
id: "UClvNJkDHdc1gvFGN_Fr_qPw",
name: "Artemis Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/rGXIwYAhI49rKBQmw_pKFMv9yEt4euHnmXOE0OOCD6ApdQXGnuPmEv7TK7cDjrjt0rUXYHuw=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/rGXIwYAhI49rKBQmw_pKFMv9yEt4euHnmXOE0OOCD6ApdQXGnuPmEv7TK7cDjrjt0rUXYHuw=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(312),
video_count: None,
short_description: "Hello and welcome to \"Artemis Music\"! Music can play an effective role in helping us lead a better and more productive life.",
)),
Channel(ChannelItem(
id: "UC5r3j8tQsB3MYZiwQFGKrdA",
name: "Disco Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/5nqhAdf26KoSKbfUB8kvhJo6rpMQw3XS345h8ZNmeXScqlB1KjJAM0T371r3QcS1mA1LZg9B1Po=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/5nqhAdf26KoSKbfUB8kvhJo6rpMQw3XS345h8ZNmeXScqlB1KjJAM0T371r3QcS1mA1LZg9B1Po=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(372),
video_count: None,
short_description: "Music is the only language in which you cannot say a mean or sarcastic thing. Have fun listening to music.",
)),
Channel(ChannelItem(
id: "UCNZYpcqym8gHcNg2GWcC6nQ",
name: "S!X - Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu_1NOzbZUJWZjtmD4NTsb9BR-TNIAzNoajv0TisvQ=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu_1NOzbZUJWZjtmD4NTsb9BR-TNIAzNoajv0TisvQ=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(178),
video_count: None,
short_description: "S!X - Music is an independent Hip-Hop label. Soundcloud : https://soundcloud.com/s1xmusic Facebook\u{a0}...",
)),
Channel(ChannelItem(
id: "UCoEryX-WO7IHBGqTAC5r9Zw",
name: "Shake Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9fMXUALsloNUJ_wLpqCS0ovprvc5W-XwfrpmWqIw=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9fMXUALsloNUJ_wLpqCS0ovprvc5W-XwfrpmWqIw=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(104),
video_count: None,
short_description: "Welcome to Shake Music, a Trap & Bass Channel / Record Label dedicated to bringing you the best tracks. All tracks on Shake\u{a0}...",
)),
Channel(ChannelItem(
id: "UCTJ9Qg-1vBu2pP_YrWUfGnQ",
name: "Miracle Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/3RMarDSmUSIexCXWCpMUkqV64uiHDXTidBLwsObHstx5-AbB8h_n8Zy1W9JymURd7ivzlDEGFw=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/3RMarDSmUSIexCXWCpMUkqV64uiHDXTidBLwsObHstx5-AbB8h_n8Zy1W9JymURd7ivzlDEGFw=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(822),
video_count: None,
short_description: "Welcome to Miracle Music! On this channel you will find a wide variety of different Deep House, Tropical House, Chill Out, EDM,.",
)),
Channel(ChannelItem(
id: "UCp6_KuNhT0kcFk-jXw9Tivg",
name: "Magic Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu-fgSc_lceD4fRL_y0b3MKd2k54DF-laDAR3Avbuw=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu-fgSc_lceD4fRL_y0b3MKd2k54DF-laDAR3Avbuw=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(462),
video_count: None,
short_description: "",
)),
Channel(ChannelItem(
id: "UCe55Gy-hFDvLZp8C8BZhBnw",
name: "Nightblue Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu-29SYt5qpqMP9Xi2A98mqL8ymI5Lg7Vzx-qpY09w=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu-29SYt5qpqMP9Xi2A98mqL8ymI5Lg7Vzx-qpY09w=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(105),
video_count: None,
short_description: "BRINGING YOU ONLY THE BEST EDM - TRAP Submit your own track for promotion here:\u{a0}...",
)),
Channel(ChannelItem(
id: "UC2fVSthyWxWSjsiEAHPzriQ",
name: "Mr_MoMo Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/7YG4jSrhx_Mfi2TsV0rJFlFARaR8kl7ilcIyzs6gSeNjwn-J88DvDWD8PSNd5o03qJRzpvhs=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/7YG4jSrhx_Mfi2TsV0rJFlFARaR8kl7ilcIyzs6gSeNjwn-J88DvDWD8PSNd5o03qJRzpvhs=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(709),
video_count: None,
short_description: "Hey there! I am Mr MoMo My channel focus on Japan music, lofi, trap & bass type beat and Japanese instrumental. I mindfully\u{a0}...",
)),
Channel(ChannelItem(
id: "UCN31w7dRjjz8CeP0GfSIo8A",
name: "Danit Music Official",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/ytc/AMLnZu9rUKtDsY-aSoE5WEwAQxvQTXiuAPYMBoJQ2mYTUA=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/ytc/AMLnZu9rUKtDsY-aSoE5WEwAQxvQTXiuAPYMBoJQ2mYTUA=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(544),
video_count: None,
short_description: "",
)),
Channel(ChannelItem(
id: "UCpEHWiTMk1eEBAdzBnAb3rA",
name: "Energy Transformation Relaxing Music ",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/RR7upyAvT7N0_qlZWfLlDSRPhLufX4W4X6-qahWvuvDCLn2cWCs0yh_HXB2iwGbk_MTwSqwWEQ=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/RR7upyAvT7N0_qlZWfLlDSRPhLufX4W4X6-qahWvuvDCLn2cWCs0yh_HXB2iwGbk_MTwSqwWEQ=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(359),
video_count: None,
short_description: "Welcome to our Energy Transformation Relaxing Music . This chakra music channel will focus on developing the best chakra\u{a0}...",
)),
Channel(ChannelItem(
id: "UCqswUMaC5yWUrkQszr8fuBA",
name: "Nonstop Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9vLN62RxNbnpa20r5XreWRlVjHXbHf7BMcvSBxoQ=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9vLN62RxNbnpa20r5XreWRlVjHXbHf7BMcvSBxoQ=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(416),
video_count: None,
short_description: "Nonstop Music - Home of 1h videos of your favourite songs and mixes. Nonstop Genres: Pop • Chillout • Tropical House • Deep\u{a0}...",
)),
Channel(ChannelItem(
id: "UChO8h2G8UjOVc081rgYU8XQ",
name: "Vibe Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9Br5pt87kuDLRFbh1MqMXeFlCLbUrwFlDIzU4s=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9Br5pt87kuDLRFbh1MqMXeFlCLbUrwFlDIzU4s=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: Verified,
subscriber_count: Some(3),
video_count: None,
short_description: "Vibe Music strives to bring the best lyric videos of popular Rap & Hip Hop songs. Be sure to Subscribe to see new videos we\u{a0}...",
)),
Channel(ChannelItem(
id: "UClV8b2EhIhIASKw-etzegyw",
name: "Suits Music",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9Aj5RtZZMdK_B_YD-8rOfi9c5ddFw5t1s4GYEeOQ=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu9Aj5RtZZMdK_B_YD-8rOfi9c5ddFw5t1s4GYEeOQ=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(120),
video_count: None,
short_description: "",
)),
Channel(ChannelItem(
id: "UCI2hwz3r5phXpOtViIA5inA",
name: "Rock Music Collection",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/kB4gWvROUIWFuJN8xwIqmPl1QV2_gXMat6COAJjXZT07E3xomc4b2JwGtDg05t1MmhgqImSifhc=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/kB4gWvROUIWFuJN8xwIqmPl1QV2_gXMat6COAJjXZT07E3xomc4b2JwGtDg05t1MmhgqImSifhc=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(817),
video_count: None,
short_description: "",
)),
Channel(ChannelItem(
id: "UC9w8My3S7h-bQZ-4R-0ZPsw",
name: "Helios Music",
avatar: [
Thumbnail(
url: "//yt3.ggpht.com/bi08T8zuYI1PlbM8M5fyZzjVvNJRJFFcQoonRQvS30opJ-OqGIq5OPrZ19qga29PIAit7OO3=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.ggpht.com/bi08T8zuYI1PlbM8M5fyZzjVvNJRJFFcQoonRQvS30opJ-OqGIq5OPrZ19qga29PIAit7OO3=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(53),
video_count: None,
short_description: "Welcome to my channel - Helios Music. I created this channel to help people have the most relaxing, refreshing and comfortable\u{a0}...",
)),
Channel(ChannelItem(
id: "UC_ODKC5gTs2LvdHXDRdDm0w",
name: "Music On",
avatar: [
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu8lUOYw4RdRwQf2Kz8RCExSmuWC78oetXF7VL67SA=s88-c-k-c0x00ffffff-no-rj-mo",
width: 88,
height: 88,
),
Thumbnail(
url: "//yt3.googleusercontent.com/ytc/AMLnZu8lUOYw4RdRwQf2Kz8RCExSmuWC78oetXF7VL67SA=s176-c-k-c0x00ffffff-no-rj-mo",
width: 176,
height: 176,
),
],
verification: None,
subscriber_count: Some(129),
video_count: None,
short_description: "Music On (UNOFFICIAL CHANNEL)",
)),
],
ctoken: Some("Eu4FEgVtdXNpYxrkBUVnSVFBa2dVZ2dFWVZVTk5kMlZRVmtoU2NFUmtabVZWWTNkMFJGcDFNa1IzZ2dFWVZVTk1lRUZUTURKbFYzWm1Xa3MwYVdOU1RucFhSRjluZ2dFWVZVTjRTM2hxVGxCNVREbFZUelZNVWxkSWVuQTFTbmhCZ2dFWVZVTkhTWGxuYVZsclMzaHVOMmMzWmtaT1JtUlljMnRuZ2dFWVZVTnNkazVLYTBSSVpHTXhaM1pHUjA1ZlJuSmZjVkIzZ2dFWVZVTTFjak5xT0hSUmMwSXpUVmxhYVhkUlJrZExjbVJCZ2dFWVZVTk9XbGx3WTNGNWJUaG5TR05PWnpKSFYyTkRObTVSZ2dFWVZVTnZSWEo1V0MxWFR6ZEpTRUpIY1ZSQlF6VnlPVnAzZ2dFWVZVTlVTamxSWnkweGRrSjFNbkJRWDFseVYxVm1SMjVSZ2dFWVZVTndObDlMZFU1b1ZEQnJZMFpyTFdwWWR6bFVhWFpuZ2dFWVZVTmxOVFZIZVMxb1JrUjJURnB3T0VNNFFscG9RbTUzZ2dFWVZVTXlabFpUZEdoNVYzaFhVMnB6YVVWQlNGQjZjbWxSZ2dFWVZVTk9NekYzTjJSU2FtcDZPRU5sVURCSFpsTkpiemhCZ2dFWVZVTndSVWhYYVZSTmF6RmxSVUpCWkhwQ2JrRmlNM0pCZ2dFWVZVTnhjM2RWVFdGRE5YbFhWWEpyVVhONmNqaG1kVUpCZ2dFWVZVTm9UemhvTWtjNFZXcFBWbU13T0RGeVoxbFZPRmhSZ2dFWVZVTnNWamhpTWtWb1NXaEpRVk5MZHkxbGRIcGxaM2wzZ2dFWVZVTkpNbWgzZWpOeU5YQm9XSEJQZEZacFNVRTFhVzVCZ2dFWVZVTTVkemhOZVROVE4yZ3RZbEZhTFRSU0xUQmFVSE4zZ2dFWVZVTmZUMFJMUXpWblZITXlUSFprU0ZoRVVtUkViVEIzc2dFR0NnUUlGUkFDGIHg6BgiC3NlYXJjaC1mZWVk"),
endpoint: search,
),
corrected_query: None,
visitor_data: None,
)

2
src/client/snapshots/rustypipe__client__search__tests__map_search_default.snap
View file

@ -23,7 +23,7 @@ SearchResult(
],
verification: Verified,
subscriber_count: Some(292),
video_count: 219,
video_count: Some(219),
short_description: "Hi, I\'m Tina, aka Doobydobap! Food is the medium I use to tell stories and connect with people who share the same passion as I\u{a0}...",
)),
Video(VideoItem(

2
src/model/mod.rs
View file

@ -946,7 +946,7 @@ pub struct ChannelItem {
/// [`None`] if hidden by the owner or not present.
pub subscriber_count: Option<u64>,
/// Number of videos from the channel
pub video_count: u64,
pub video_count: Option<u64>,
/// Abbreviated channel description
pub short_description: String,
}

26
src/timeago.rs
View file

@ -192,6 +192,30 @@ pub fn parse_timeago_to_dt(lang: Language, textual_date: &str) -> Option<OffsetD
parse_timeago(lang, textual_date).map(|ta| ta.into())
}
pub(crate) fn parse_timeago_or_warn(
lang: Language,
textual_date: &str,
warnings: &mut Vec<String>,
) -> Option<TimeAgo> {
let res = parse_timeago(lang, textual_date);
if res.is_none() {
warnings.push(format!("could not parse timeago `{}`", textual_date));
}
res
}
pub(crate) fn parse_timeago_dt_or_warn(
lang: Language,
textual_date: &str,
warnings: &mut Vec<String>,
) -> Option<OffsetDateTime> {
let res = parse_timeago_to_dt(lang, textual_date);
if res.is_none() {
warnings.push(format!("could not parse timeago `{}`", textual_date));
}
res
}
/// Parse a textual date (e.g. "29 minutes ago" or "Jul 2, 2014") into a ParsedDate object.
///
/// Returns None if the date could not be parsed.
@ -254,7 +278,7 @@ pub(crate) fn parse_textual_date_or_warn(
) -> Option<OffsetDateTime> {
let res = parse_textual_date_to_dt(lang, textual_date);
if res.is_none() {
warnings.push(format!("could not parse timeago `{}`", textual_date));
warnings.push(format!("could not parse textual date `{}`", textual_date));
}
res
}

5037
testfiles/search/20221121_AB3_channel_handles.json Normal file
View file

File diff suppressed because it is too large Load diff

7
tests/youtube.rs
View file

@ -17,6 +17,7 @@ use rustypipe::model::{
use rustypipe::param::search_filter::{self, SearchFilter};
const VISITOR_DATA_3TAB_CHANNEL_LAYOUT: &str = "CgtOa256ckVkcG5YVSiirbyaBg%3D%3D";
const VISITOR_DATA_SEARCH_CHANNEL_HANDLES: &str = "CgszYlc1Yk1WZGRCSSjrwOSbBg%3D%3D";
//#PLAYER
@ -1105,14 +1106,14 @@ async fn search() {
#[rstest]
#[case::video(search_filter::Entity::Video)]
#[case::video(search_filter::Entity::Channel)]
#[case::video(search_filter::Entity::Playlist)]
#[case::channel(search_filter::Entity::Channel)]
#[case::playlist(search_filter::Entity::Playlist)]
#[tokio::test]
async fn search_filter_entity(#[case] entity: search_filter::Entity) {
let rp = RustyPipe::builder().strict().build();
let mut result = rp
.query()
.search_filter("music", &SearchFilter::new().entity(entity))
.search_filter("with no videos", &SearchFilter::new().entity(entity))
.await
.unwrap();