Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: add support for A/B test 7 (short date format)

Browse source
This commit is contained in:
ThetaDev 2023-05-28 21:07:03 +02:00
parent cca9838b7e
commit cc2cadc309
21 changed files with 184 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

21
codegen/src/abtest.rs
View file

@ -4,6 +4,8 @@ use anyhow::{bail, Result};
use futures::{stream, StreamExt};
use indicatif::{ProgressBar, ProgressStyle};
use num_enum::TryFromPrimitive;
use once_cell::sync::Lazy;
use regex::Regex;
use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext};
use rustypipe::model::YouTubeItem;
use rustypipe::param::search_filter::{ItemType, SearchFilter};
@ -21,6 +23,7 @@ pub enum ABTest {
TrendsVideoTab = 4,
TrendsPageHeaderRenderer = 5,
DiscographyPage = 6,
ShortDateFormat = 7,
}
const TESTS_TO_RUN: [ABTest; 3] = [
@ -90,6 +93,7 @@ pub async fn run_test(
ABTest::TrendsVideoTab => trends_video_tab(&query).await,
ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await,
ABTest::DiscographyPage => discography_page(&query).await,
ABTest::ShortDateFormat => short_date_format(&query).await,
}
.unwrap();
pb.inc(1);
@ -223,10 +227,19 @@ pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result<bool> {
}
pub async fn discography_page(rp: &RustyPipeQuery) -> Result<bool> {
let artist = rp
.music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false)
.await
.unwrap();
let artist = rp.music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false).await?;
Ok(artist.albums.len() <= 10)
}
pub async fn short_date_format(rp: &RustyPipeQuery) -> Result<bool> {
static SHORT_DATE: Lazy<Regex> = Lazy::new(|| Regex::new("\\d(?:y|mo|w|d|h|min) ").unwrap());
let channel = rp.channel_videos("UC2DjFE7Xf11URZqWBigcVOQ").await?;
Ok(channel.content.items.iter().any(|itm| {
itm.publish_date_txt
.as_deref()
.map(|d| SHORT_DATE.is_match(d))
.unwrap_or_default()
}))
}

8
notes/AB_Tests.md
View file

@ -376,3 +376,11 @@ visitor data cookie to be set, as it was the case with the old system.
**NEW**
![A/B test 4 old screenshot](./_img/ab_6_new.png)
## [7] Short timeago format
- **Encountered on:** 28.05.2023
- **Impact:** 🟡 Medium
YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to
a short format (*21h ago*, *3d ago*).

28
src/client/channel.rs
View file

@ -200,15 +200,20 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<Paginator<VideoItem>>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?;
let visitor_data = self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let channel_data = map_channel(
MapChannelData {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data.clone(),
visitor_data: visitor_data.clone(),
has_shorts: content.has_shorts,
has_live: content.has_live,
},
@ -226,7 +231,7 @@ impl MapResponse<Channel<Paginator<VideoItem>>> for response::Channel {
None,
mapper.items,
mapper.ctoken,
self.response_context.visitor_data,
visitor_data,
crate::model::paginator::ContinuationEndpoint::Browse,
);
@ -243,15 +248,20 @@ impl MapResponse<Channel<Paginator<PlaylistItem>>> for response::Channel {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<Paginator<PlaylistItem>>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?;
let visitor_data = self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let channel_data = map_channel(
MapChannelData {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data,
visitor_data,
has_shorts: content.has_shorts,
has_live: content.has_live,
},
@ -280,6 +290,7 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Channel<ChannelInfo>>, ExtractionError> {
let content = map_channel_content(id, self.contents, self.alerts)?;
let channel_data = map_channel(
@ -287,7 +298,10 @@ impl MapResponse<Channel<ChannelInfo>> for response::Channel {
header: self.header,
metadata: self.metadata,
microformat: self.microformat,
visitor_data: self.response_context.visitor_data,
visitor_data: self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
has_shorts: content.has_shorts,
has_live: content.has_live,
},
@ -605,7 +619,7 @@ mod tests {
let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<Paginator<VideoItem>>> =
channel.map_response(id, Language::En, None).unwrap();
channel.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -632,7 +646,7 @@ mod tests {
let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<Paginator<PlaylistItem>>> = channel
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None)
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None)
.unwrap();
assert!(
@ -651,7 +665,7 @@ mod tests {
let channel: response::Channel =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Channel<ChannelInfo>> = channel
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None)
.map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None)
.unwrap();
assert!(

9
src/client/mod.rs
View file

@ -1247,7 +1247,12 @@ impl RustyPipeQuery {
})
} else {
match serde_json::from_str::<R>(&body) {
Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) {
Ok(deserialized) => match deserialized.map_response(
id,
self.opts.lang,
deobf,
self.opts.visitor_data.as_deref(),
) {
Ok(mapres) => Ok(mapres),
Err(e) => Err(e.into()),
},
@ -1453,11 +1458,13 @@ trait MapResponse<T> {
/// that the returned entity matches this ID and return an error instead.
/// - `lang`: Language of the request. Used for mapping localized information like dates.
/// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method)
/// - `vdata`: Visitor data option of the client
fn map_response(
self,
id: &str,
lang: Language,
deobf: Option<&DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<T>, ExtractionError>;
}

11
src/client/music_artist.rs
View file

@ -96,6 +96,7 @@ impl MapResponse<MusicArtist> for response::MusicArtist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicArtist>, ExtractionError> {
let mapped = map_artist_page(self, id, lang, false)?;
Ok(MapResult {
@ -111,6 +112,7 @@ impl MapResponse<(MusicArtist, bool)> for response::MusicArtist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<(MusicArtist, bool)>, ExtractionError> {
map_artist_page(self, id, lang, true)
}
@ -286,6 +288,7 @@ impl MapResponse<Vec<AlbumItem>> for response::MusicArtistAlbums {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Vec<AlbumItem>>, ExtractionError> {
// dbg!(&self);
@ -356,7 +359,7 @@ mod tests {
let resp: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<(MusicArtist, bool)> =
resp.map_response(id, Language::En, None).unwrap();
resp.map_response(id, Language::En, None, None).unwrap();
let (mut artist, can_fetch_more) = map_res.c;
assert!(
@ -371,7 +374,7 @@ mod tests {
let resp: response::MusicArtistAlbums =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let mut map_res: MapResult<Vec<AlbumItem>> =
resp.map_response(id, Language::En, None).unwrap();
resp.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -392,7 +395,7 @@ mod tests {
let artist: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicArtist> = artist
.map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None)
.map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None, None)
.unwrap();
assert!(
@ -411,7 +414,7 @@ mod tests {
let artist: response::MusicArtist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let res: Result<MapResult<MusicArtist>, ExtractionError> =
artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None);
artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None, None);
let e = res.unwrap_err();
match e {

4
src/client/music_charts.rs
View file

@ -60,6 +60,7 @@ impl MapResponse<MusicCharts> for response::MusicCharts {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<MusicCharts>, crate::error::ExtractionError> {
let countries = self
.framework_updates
@ -164,7 +165,8 @@ mod tests {
let charts: response::MusicCharts =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicCharts> = charts.map_response("", Language::En, None).unwrap();
let map_res: MapResult<MusicCharts> =
charts.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

13
src/client/music_details.rs
View file

@ -157,6 +157,7 @@ impl MapResponse<TrackDetails> for response::MusicDetails {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<TrackDetails>, ExtractionError> {
let tabs = self
.contents
@ -237,6 +238,7 @@ impl MapResponse<Paginator<TrackItem>> for response::MusicDetails {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<TrackItem>>, ExtractionError> {
let tabs = self
.contents
@ -297,6 +299,7 @@ impl MapResponse<Lyrics> for response::MusicLyrics {
id: &str,
_lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Lyrics>, ExtractionError> {
let lyrics = self
.contents
@ -330,6 +333,7 @@ impl MapResponse<MusicRelated> for response::MusicRelated {
_id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicRelated>, ExtractionError> {
// Find artist
let artist_id = self
@ -422,7 +426,7 @@ mod tests {
let details: response::MusicDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::TrackDetails> =
details.map_response(id, Language::En, None).unwrap();
details.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -442,7 +446,7 @@ mod tests {
let radio: response::MusicDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<TrackItem>> =
radio.map_response(id, Language::En, None).unwrap();
radio.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -459,7 +463,7 @@ mod tests {
let lyrics: response::MusicLyrics =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Lyrics> = lyrics.map_response("", Language::En, None).unwrap();
let map_res: MapResult<Lyrics> = lyrics.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -476,7 +480,8 @@ mod tests {
let lyrics: response::MusicRelated =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicRelated> = lyrics.map_response("", Language::En, None).unwrap();
let map_res: MapResult<MusicRelated> =
lyrics.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

6
src/client/music_genres.rs
View file

@ -57,6 +57,7 @@ impl MapResponse<Vec<MusicGenreItem>> for response::MusicGenres {
_id: &str,
_lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<Vec<MusicGenreItem>>, ExtractionError> {
let content = self
.contents
@ -110,6 +111,7 @@ impl MapResponse<MusicGenre> for response::MusicGenre {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<MusicGenre>, ExtractionError> {
// dbg!(&self);
@ -214,7 +216,7 @@ mod tests {
let playlist: response::MusicGenres =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<model::MusicGenreItem>> =
playlist.map_response("", Language::En, None).unwrap();
playlist.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -234,7 +236,7 @@ mod tests {
let playlist: response::MusicGenre =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicGenre> =
playlist.map_response(id, Language::En, None).unwrap();
playlist.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

11
src/client/music_new.rs
View file

@ -52,6 +52,7 @@ impl<T: FromYtItem> MapResponse<Vec<T>> for response::MusicNew {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<crate::serializer::MapResult<Vec<T>>, ExtractionError> {
let items = self
.contents
@ -96,8 +97,9 @@ mod tests {
let new_albums: response::MusicNew =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<AlbumItem>> =
new_albums.map_response("", Language::En, None).unwrap();
let map_res: MapResult<Vec<AlbumItem>> = new_albums
.map_response("", Language::En, None, None)
.unwrap();
assert!(
map_res.warnings.is_empty(),
@ -115,8 +117,9 @@ mod tests {
let new_albums: response::MusicNew =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<TrackItem>> =
new_albums.map_response("", Language::En, None).unwrap();
let map_res: MapResult<Vec<TrackItem>> = new_albums
.map_response("", Language::En, None, None)
.unwrap();
assert!(
map_res.warnings.is_empty(),

6
src/client/music_playlist.rs
View file

@ -122,6 +122,7 @@ impl MapResponse<MusicPlaylist> for response::MusicPlaylist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicPlaylist>, ExtractionError> {
// dbg!(&self);
@ -267,6 +268,7 @@ impl MapResponse<MusicAlbum> for response::MusicPlaylist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicAlbum>, ExtractionError> {
// dbg!(&self);
@ -418,7 +420,7 @@ mod tests {
let playlist: response::MusicPlaylist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicPlaylist> =
playlist.map_response(id, Language::En, None).unwrap();
playlist.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -443,7 +445,7 @@ mod tests {
let playlist: response::MusicPlaylist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<model::MusicAlbum> =
playlist.map_response(id, Language::En, None).unwrap();
playlist.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

18
src/client/music_search.rs
View file

@ -231,6 +231,7 @@ impl MapResponse<MusicSearchResult> for response::MusicSearch {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchResult>, crate::error::ExtractionError> {
// dbg!(&self);
@ -296,6 +297,7 @@ impl<T: FromYtItem> MapResponse<MusicSearchFiltered<T>> for response::MusicSearc
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchFiltered<T>>, ExtractionError> {
// dbg!(&self);
@ -356,6 +358,7 @@ impl MapResponse<MusicSearchSuggestion> for response::MusicSearchSuggestion {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<MusicSearchSuggestion>, ExtractionError> {
let mut mapper = MusicListMapper::new(lang);
let mut terms = Vec::new();
@ -419,7 +422,7 @@ mod tests {
let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchResult> =
search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -442,7 +445,7 @@ mod tests {
let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<TrackItem>> =
search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -461,7 +464,7 @@ mod tests {
let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<AlbumItem>> =
search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -480,7 +483,7 @@ mod tests {
let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<ArtistItem>> =
search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -501,7 +504,7 @@ mod tests {
let search: response::MusicSearch =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchFiltered<MusicPlaylistItem>> =
search.map_response("", Language::En, None).unwrap();
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -521,8 +524,9 @@ mod tests {
let suggestion: response::MusicSearchSuggestion =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<MusicSearchSuggestion> =
suggestion.map_response("", Language::En, None).unwrap();
let map_res: MapResult<MusicSearchSuggestion> = suggestion
.map_response("", Language::En, None, None)
.unwrap();
assert!(
map_res.warnings.is_empty(),

12
src/client/pagination.rs
View file

@ -96,6 +96,7 @@ impl MapResponse<Paginator<YouTubeItem>> for response::Continuation {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<YouTubeItem>>, ExtractionError> {
let items = self
.on_response_received_actions
@ -131,6 +132,7 @@ impl MapResponse<Paginator<MusicItem>> for response::MusicContinuation {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<MusicItem>>, ExtractionError> {
let mut mapper = MusicListMapper::new(lang);
let mut continuations = Vec::new();
@ -353,7 +355,7 @@ mod tests {
let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None).unwrap();
items.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -375,7 +377,7 @@ mod tests {
let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None).unwrap();
items.map_response("", Language::En, None, None).unwrap();
let paginator: Paginator<VideoItem> =
map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse);
@ -398,7 +400,7 @@ mod tests {
let items: response::Continuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<YouTubeItem>> =
items.map_response("", Language::En, None).unwrap();
items.map_response("", Language::En, None, None).unwrap();
let paginator: Paginator<PlaylistItem> =
map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse);
@ -421,7 +423,7 @@ mod tests {
let items: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> =
items.map_response("", Language::En, None).unwrap();
items.map_response("", Language::En, None, None).unwrap();
let paginator: Paginator<TrackItem> =
map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);
@ -442,7 +444,7 @@ mod tests {
let items: response::MusicContinuation =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<MusicItem>> =
items.map_response("", Language::En, None).unwrap();
items.map_response("", Language::En, None, None).unwrap();
let paginator: Paginator<MusicPlaylistItem> =
map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse);

8
src/client/player.rs
View file

@ -143,6 +143,7 @@ impl MapResponse<VideoPlayer> for response::Player {
id: &str,
_lang: Language,
deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<super::MapResult<VideoPlayer>, ExtractionError> {
let deobf = Deobfuscator::new(deobf.unwrap())?;
let mut warnings = vec![];
@ -372,7 +373,10 @@ impl MapResponse<VideoPlayer> for response::Player {
hls_manifest_url: streaming_data.hls_manifest_url,
dash_manifest_url: streaming_data.dash_manifest_url,
preview_frames,
visitor_data: self.response_context.visitor_data,
visitor_data: self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
},
warnings,
})
@ -717,7 +721,7 @@ mod tests {
let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = resp
.map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA))
.map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA), None)
.unwrap();
assert!(

8
src/client/playlist.rs
View file

@ -37,6 +37,7 @@ impl MapResponse<Playlist> for response::Playlist {
id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Playlist>, ExtractionError> {
let (Some(contents), Some(header)) = (self.contents, self.header) else {
return Err(response::alerts_to_err(id, self.alerts));
@ -152,7 +153,10 @@ impl MapResponse<Playlist> for response::Playlist {
channel,
last_update,
last_update_txt,
visitor_data: self.response_context.visitor_data,
visitor_data: self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
},
warnings: mapper.warnings,
})
@ -181,7 +185,7 @@ mod tests {
let playlist: response::Playlist =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = playlist.map_response(id, Language::En, None).unwrap();
let map_res = playlist.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

9
src/client/search.rs
View file

@ -92,6 +92,7 @@ impl MapResponse<SearchResult> for response::Search {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<SearchResult>, ExtractionError> {
let items = self
.contents
@ -113,7 +114,10 @@ impl MapResponse<SearchResult> for response::Search {
crate::model::paginator::ContinuationEndpoint::Search,
),
corrected_query: mapper.corrected_query,
visitor_data: self.response_context.visitor_data,
visitor_data: self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
},
warnings: mapper.warnings,
})
@ -145,7 +149,8 @@ mod tests {
let json_file = File::open(json_path).unwrap();
let search: response::Search = serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<SearchResult> = search.map_response("", Language::En, None).unwrap();
let map_res: MapResult<SearchResult> =
search.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

16
src/client/trends.rs
View file

@ -54,6 +54,7 @@ impl MapResponse<Paginator<VideoItem>> for response::Startpage {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<Paginator<VideoItem>>, ExtractionError> {
let grid = self
.contents
@ -70,7 +71,9 @@ impl MapResponse<Paginator<VideoItem>> for response::Startpage {
Ok(map_startpage_videos(
grid,
lang,
self.response_context.visitor_data,
self.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned)),
))
}
}
@ -81,6 +84,7 @@ impl MapResponse<Vec<VideoItem>> for response::Trending {
_id: &str,
lang: crate::param::Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Vec<VideoItem>>, ExtractionError> {
let items = self
.contents
@ -146,8 +150,9 @@ mod tests {
let startpage: response::Startpage =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Paginator<VideoItem>> =
startpage.map_response("", Language::En, None).unwrap();
let map_res: MapResult<Paginator<VideoItem>> = startpage
.map_response("", Language::En, None, None)
.unwrap();
assert!(
map_res.warnings.is_empty(),
@ -169,8 +174,9 @@ mod tests {
let startpage: response::Trending =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res: MapResult<Vec<VideoItem>> =
startpage.map_response("", Language::En, None).unwrap();
let map_res: MapResult<Vec<VideoItem>> = startpage
.map_response("", Language::En, None, None)
.unwrap();
assert!(
map_res.warnings.is_empty(),

1
src/client/url_resolver.rs
View file

@ -328,6 +328,7 @@ impl MapResponse<UrlTarget> for response::ResolvedUrl {
_id: &str,
_lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<UrlTarget>, ExtractionError> {
let pt = self.endpoint.page_type();
if let NavigationEndpoint::Browse {

15
src/client/video_details.rs
View file

@ -82,6 +82,7 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
vdata: Option<&str>,
) -> Result<MapResult<VideoDetails>, ExtractionError> {
let mut warnings = Vec::new();
@ -256,7 +257,10 @@ impl MapResponse<VideoDetails> for response::VideoDetails {
_ => return Err(ExtractionError::InvalidData("invalid channel link".into())),
};
let visitor_data = self.response_context.visitor_data;
let visitor_data = self
.response_context
.visitor_data
.or_else(|| vdata.map(str::to_owned));
let recommended = contents
.two_column_watch_next_results
.secondary_results
@ -369,6 +373,7 @@ impl MapResponse<Paginator<Comment>> for response::VideoComments {
_id: &str,
lang: Language,
_deobf: Option<&crate::deobfuscate::DeobfData>,
_vdata: Option<&str>,
) -> Result<MapResult<Paginator<Comment>>, ExtractionError> {
let received_endpoints = self.on_response_received_endpoints;
let mut warnings = received_endpoints.warnings;
@ -561,7 +566,7 @@ mod tests {
let details: response::VideoDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = details.map_response(id, Language::En, None).unwrap();
let map_res = details.map_response(id, Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),
@ -581,7 +586,9 @@ mod tests {
let details: response::VideoDetails =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let err = details.map_response("", Language::En, None).unwrap_err();
let err = details
.map_response("", Language::En, None, None)
.unwrap_err();
assert!(matches!(
err,
crate::error::ExtractionError::NotFound { .. }
@ -597,7 +604,7 @@ mod tests {
let comments: response::VideoComments =
serde_json::from_reader(BufReader::new(json_file)).unwrap();
let map_res = comments.map_response("", Language::En, None).unwrap();
let map_res = comments.map_response("", Language::En, None, None).unwrap();
assert!(
map_res.warnings.is_empty(),

35
src/util/dictionary.rs
View file

@ -1324,27 +1324,36 @@ pub(crate) fn entry(lang: Language) -> Entry {
},
Language::En | Language::EnGb | Language::EnIn => Entry {
timeago_tokens: ::phf::Map {
key: 7485420634051515786,
key: 12913932095322966823,
disps: &[
(0, 7),
(8, 12),
(5, 0),
(2, 0),
(2, 0),
(5, 18),
(4, 15),
(2, 17),
],
entries: &[
("months", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
("month", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
("hours", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
("days", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
("years", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
("week", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
("second", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
("day", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
("month", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
("minutes", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
("min", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
("y", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
("days", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
("minute", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
("seconds", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
("s", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
("mo", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
("hours", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
("w", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
("h", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
("d", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
("second", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
("years", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
("months", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
("week", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
("weeks", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
("hour", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
("minutes", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
("year", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
("hour", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
],
},
date_order: &[DateCmp::D, DateCmp::Y],

9
testfiles/dict/dictionary.json
View file

@ -754,7 +754,14 @@
"week": "W",
"weeks": "W",
"year": "Y",
"years": "Y"
"years": "Y",
"y": "Y",
"mo": "M",
"w": "W",
"d": "D",
"h": "h",
"min": "m",
"s": "s"
},
"date_order": "DY",
"months": {

7
tests/youtube.rs
View file

@ -2336,7 +2336,12 @@ fn lang() -> Language {
/// Get a new RustyPipe instance
#[fixture]
fn rp(lang: Language) -> RustyPipe {
RustyPipe::builder().strict().lang(lang).build()
let vdata = std::env::var("YT_VDATA").ok();
RustyPipe::builder()
.strict()
.lang(lang)
.visitor_data_opt(vdata)
.build()
}
/// Get a flag signaling if the language is set to English