From cc2cadc3093ea5c53acbd06b6fa531ee2b596972 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Sun, 28 May 2023 21:07:03 +0200 Subject: [PATCH] fix: add support for A/B test 7 (short date format) --- codegen/src/abtest.rs | 21 ++++++++++++++++---- notes/AB_Tests.md | 8 ++++++++ src/client/channel.rs | 28 ++++++++++++++++++++------- src/client/mod.rs | 9 ++++++++- src/client/music_artist.rs | 11 +++++++---- src/client/music_charts.rs | 4 +++- src/client/music_details.rs | 13 +++++++++---- src/client/music_genres.rs | 6 ++++-- src/client/music_new.rs | 11 +++++++---- src/client/music_playlist.rs | 6 ++++-- src/client/music_search.rs | 18 ++++++++++------- src/client/pagination.rs | 12 +++++++----- src/client/player.rs | 8 ++++++-- src/client/playlist.rs | 8 ++++++-- src/client/search.rs | 9 +++++++-- src/client/trends.rs | 16 +++++++++++----- src/client/url_resolver.rs | 1 + src/client/video_details.rs | 15 +++++++++++---- src/util/dictionary.rs | 35 +++++++++++++++++++++------------- testfiles/dict/dictionary.json | 9 ++++++++- tests/youtube.rs | 7 ++++++- 21 files changed, 184 insertions(+), 71 deletions(-) diff --git a/codegen/src/abtest.rs b/codegen/src/abtest.rs index ac13a8c..9acf827 100644 --- a/codegen/src/abtest.rs +++ b/codegen/src/abtest.rs @@ -4,6 +4,8 @@ use anyhow::{bail, Result}; use futures::{stream, StreamExt}; use indicatif::{ProgressBar, ProgressStyle}; use num_enum::TryFromPrimitive; +use once_cell::sync::Lazy; +use regex::Regex; use rustypipe::client::{ClientType, RustyPipe, RustyPipeQuery, YTContext}; use rustypipe::model::YouTubeItem; use rustypipe::param::search_filter::{ItemType, SearchFilter}; @@ -21,6 +23,7 @@ pub enum ABTest { TrendsVideoTab = 4, TrendsPageHeaderRenderer = 5, DiscographyPage = 6, + ShortDateFormat = 7, } const TESTS_TO_RUN: [ABTest; 3] = [ @@ -90,6 +93,7 @@ pub async fn run_test( ABTest::TrendsVideoTab => trends_video_tab(&query).await, ABTest::TrendsPageHeaderRenderer => trends_page_header_renderer(&query).await, ABTest::DiscographyPage => discography_page(&query).await, + ABTest::ShortDateFormat => short_date_format(&query).await, } .unwrap(); pb.inc(1); @@ -223,10 +227,19 @@ pub async fn trends_page_header_renderer(rp: &RustyPipeQuery) -> Result { } pub async fn discography_page(rp: &RustyPipeQuery) -> Result { - let artist = rp - .music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false) - .await - .unwrap(); + let artist = rp.music_artist("UC7cl4MmM6ZZ2TcFyMk_b4pg", false).await?; Ok(artist.albums.len() <= 10) } + +pub async fn short_date_format(rp: &RustyPipeQuery) -> Result { + static SHORT_DATE: Lazy = Lazy::new(|| Regex::new("\\d(?:y|mo|w|d|h|min) ").unwrap()); + let channel = rp.channel_videos("UC2DjFE7Xf11URZqWBigcVOQ").await?; + + Ok(channel.content.items.iter().any(|itm| { + itm.publish_date_txt + .as_deref() + .map(|d| SHORT_DATE.is_match(d)) + .unwrap_or_default() + })) +} diff --git a/notes/AB_Tests.md b/notes/AB_Tests.md index 9a2fcf8..b9ffbf0 100644 --- a/notes/AB_Tests.md +++ b/notes/AB_Tests.md @@ -376,3 +376,11 @@ visitor data cookie to be set, as it was the case with the old system. **NEW** ![A/B test 4 old screenshot](./_img/ab_6_new.png) + +## [7] Short timeago format + +- **Encountered on:** 28.05.2023 +- **Impact:** 🟡 Medium + +YouTube changed their date format from the long format (*21 hours ago*, *3 days ago*) to +a short format (*21h ago*, *3d ago*). diff --git a/src/client/channel.rs b/src/client/channel.rs index 867cc55..7ec1be5 100644 --- a/src/client/channel.rs +++ b/src/client/channel.rs @@ -200,15 +200,20 @@ impl MapResponse>> for response::Channel { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result>>, ExtractionError> { let content = map_channel_content(id, self.contents, self.alerts)?; + let visitor_data = self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)); let channel_data = map_channel( MapChannelData { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data: self.response_context.visitor_data.clone(), + visitor_data: visitor_data.clone(), has_shorts: content.has_shorts, has_live: content.has_live, }, @@ -226,7 +231,7 @@ impl MapResponse>> for response::Channel { None, mapper.items, mapper.ctoken, - self.response_context.visitor_data, + visitor_data, crate::model::paginator::ContinuationEndpoint::Browse, ); @@ -243,15 +248,20 @@ impl MapResponse>> for response::Channel { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result>>, ExtractionError> { let content = map_channel_content(id, self.contents, self.alerts)?; + let visitor_data = self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)); let channel_data = map_channel( MapChannelData { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data: self.response_context.visitor_data, + visitor_data, has_shorts: content.has_shorts, has_live: content.has_live, }, @@ -280,6 +290,7 @@ impl MapResponse> for response::Channel { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result>, ExtractionError> { let content = map_channel_content(id, self.contents, self.alerts)?; let channel_data = map_channel( @@ -287,7 +298,10 @@ impl MapResponse> for response::Channel { header: self.header, metadata: self.metadata, microformat: self.microformat, - visitor_data: self.response_context.visitor_data, + visitor_data: self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)), has_shorts: content.has_shorts, has_live: content.has_live, }, @@ -605,7 +619,7 @@ mod tests { let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult>> = - channel.map_response(id, Language::En, None).unwrap(); + channel.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -632,7 +646,7 @@ mod tests { let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult>> = channel - .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None) + .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None) .unwrap(); assert!( @@ -651,7 +665,7 @@ mod tests { let channel: response::Channel = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = channel - .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None) + .map_response("UC2DjFE7Xf11URZqWBigcVOQ", Language::En, None, None) .unwrap(); assert!( diff --git a/src/client/mod.rs b/src/client/mod.rs index 5b9398c..22fc832 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -1247,7 +1247,12 @@ impl RustyPipeQuery { }) } else { match serde_json::from_str::(&body) { - Ok(deserialized) => match deserialized.map_response(id, self.opts.lang, deobf) { + Ok(deserialized) => match deserialized.map_response( + id, + self.opts.lang, + deobf, + self.opts.visitor_data.as_deref(), + ) { Ok(mapres) => Ok(mapres), Err(e) => Err(e.into()), }, @@ -1453,11 +1458,13 @@ trait MapResponse { /// that the returned entity matches this ID and return an error instead. /// - `lang`: Language of the request. Used for mapping localized information like dates. /// - `deobf`: Deobfuscator (if passed to the `execute_request_deobf` method) + /// - `vdata`: Visitor data option of the client fn map_response( self, id: &str, lang: Language, deobf: Option<&DeobfData>, + vdata: Option<&str>, ) -> Result, ExtractionError>; } diff --git a/src/client/music_artist.rs b/src/client/music_artist.rs index d5f2c64..75dafb5 100644 --- a/src/client/music_artist.rs +++ b/src/client/music_artist.rs @@ -96,6 +96,7 @@ impl MapResponse for response::MusicArtist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { let mapped = map_artist_page(self, id, lang, false)?; Ok(MapResult { @@ -111,6 +112,7 @@ impl MapResponse<(MusicArtist, bool)> for response::MusicArtist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { map_artist_page(self, id, lang, true) } @@ -286,6 +288,7 @@ impl MapResponse> for response::MusicArtistAlbums { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { // dbg!(&self); @@ -356,7 +359,7 @@ mod tests { let resp: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult<(MusicArtist, bool)> = - resp.map_response(id, Language::En, None).unwrap(); + resp.map_response(id, Language::En, None, None).unwrap(); let (mut artist, can_fetch_more) = map_res.c; assert!( @@ -371,7 +374,7 @@ mod tests { let resp: response::MusicArtistAlbums = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let mut map_res: MapResult> = - resp.map_response(id, Language::En, None).unwrap(); + resp.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -392,7 +395,7 @@ mod tests { let artist: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = artist - .map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None) + .map_response("UClmXPfaYhXOYsNn_QUyheWQ", Language::En, None, None) .unwrap(); assert!( @@ -411,7 +414,7 @@ mod tests { let artist: response::MusicArtist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let res: Result, ExtractionError> = - artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None); + artist.map_response("UCLkAepWjdylmXSltofFvsYQ", Language::En, None, None); let e = res.unwrap_err(); match e { diff --git a/src/client/music_charts.rs b/src/client/music_charts.rs index 1176dfc..87bc5a2 100644 --- a/src/client/music_charts.rs +++ b/src/client/music_charts.rs @@ -60,6 +60,7 @@ impl MapResponse for response::MusicCharts { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, crate::error::ExtractionError> { let countries = self .framework_updates @@ -164,7 +165,8 @@ mod tests { let charts: response::MusicCharts = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = charts.map_response("", Language::En, None).unwrap(); + let map_res: MapResult = + charts.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_details.rs b/src/client/music_details.rs index 06578f9..9a6d620 100644 --- a/src/client/music_details.rs +++ b/src/client/music_details.rs @@ -157,6 +157,7 @@ impl MapResponse for response::MusicDetails { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { let tabs = self .contents @@ -237,6 +238,7 @@ impl MapResponse> for response::MusicDetails { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let tabs = self .contents @@ -297,6 +299,7 @@ impl MapResponse for response::MusicLyrics { id: &str, _lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { let lyrics = self .contents @@ -330,6 +333,7 @@ impl MapResponse for response::MusicRelated { _id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { // Find artist let artist_id = self @@ -422,7 +426,7 @@ mod tests { let details: response::MusicDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - details.map_response(id, Language::En, None).unwrap(); + details.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -442,7 +446,7 @@ mod tests { let radio: response::MusicDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - radio.map_response(id, Language::En, None).unwrap(); + radio.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -459,7 +463,7 @@ mod tests { let lyrics: response::MusicLyrics = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = lyrics.map_response("", Language::En, None).unwrap(); + let map_res: MapResult = lyrics.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -476,7 +480,8 @@ mod tests { let lyrics: response::MusicRelated = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = lyrics.map_response("", Language::En, None).unwrap(); + let map_res: MapResult = + lyrics.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_genres.rs b/src/client/music_genres.rs index 3a696ca..f62273d 100644 --- a/src/client/music_genres.rs +++ b/src/client/music_genres.rs @@ -57,6 +57,7 @@ impl MapResponse> for response::MusicGenres { _id: &str, _lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let content = self .contents @@ -110,6 +111,7 @@ impl MapResponse for response::MusicGenre { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { // dbg!(&self); @@ -214,7 +216,7 @@ mod tests { let playlist: response::MusicGenres = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - playlist.map_response("", Language::En, None).unwrap(); + playlist.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -234,7 +236,7 @@ mod tests { let playlist: response::MusicGenre = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(id, Language::En, None).unwrap(); + playlist.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_new.rs b/src/client/music_new.rs index e80bff1..7abb1fa 100644 --- a/src/client/music_new.rs +++ b/src/client/music_new.rs @@ -52,6 +52,7 @@ impl MapResponse> for response::MusicNew { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let items = self .contents @@ -96,8 +97,9 @@ mod tests { let new_albums: response::MusicNew = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - new_albums.map_response("", Language::En, None).unwrap(); + let map_res: MapResult> = new_albums + .map_response("", Language::En, None, None) + .unwrap(); assert!( map_res.warnings.is_empty(), @@ -115,8 +117,9 @@ mod tests { let new_albums: response::MusicNew = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - new_albums.map_response("", Language::En, None).unwrap(); + let map_res: MapResult> = new_albums + .map_response("", Language::En, None, None) + .unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_playlist.rs b/src/client/music_playlist.rs index 0bea1b3..41f8974 100644 --- a/src/client/music_playlist.rs +++ b/src/client/music_playlist.rs @@ -122,6 +122,7 @@ impl MapResponse for response::MusicPlaylist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { // dbg!(&self); @@ -267,6 +268,7 @@ impl MapResponse for response::MusicPlaylist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { // dbg!(&self); @@ -418,7 +420,7 @@ mod tests { let playlist: response::MusicPlaylist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(id, Language::En, None).unwrap(); + playlist.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -443,7 +445,7 @@ mod tests { let playlist: response::MusicPlaylist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - playlist.map_response(id, Language::En, None).unwrap(); + playlist.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/music_search.rs b/src/client/music_search.rs index eafb524..3e2389d 100644 --- a/src/client/music_search.rs +++ b/src/client/music_search.rs @@ -231,6 +231,7 @@ impl MapResponse for response::MusicSearch { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, crate::error::ExtractionError> { // dbg!(&self); @@ -296,6 +297,7 @@ impl MapResponse> for response::MusicSearc _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { // dbg!(&self); @@ -356,6 +358,7 @@ impl MapResponse for response::MusicSearchSuggestion { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { let mut mapper = MusicListMapper::new(lang); let mut terms = Vec::new(); @@ -419,7 +422,7 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult = - search.map_response("", Language::En, None).unwrap(); + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -442,7 +445,7 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - search.map_response("", Language::En, None).unwrap(); + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -461,7 +464,7 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - search.map_response("", Language::En, None).unwrap(); + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -480,7 +483,7 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - search.map_response("", Language::En, None).unwrap(); + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -501,7 +504,7 @@ mod tests { let search: response::MusicSearch = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - search.map_response("", Language::En, None).unwrap(); + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -521,8 +524,9 @@ mod tests { let suggestion: response::MusicSearchSuggestion = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = - suggestion.map_response("", Language::En, None).unwrap(); + let map_res: MapResult = suggestion + .map_response("", Language::En, None, None) + .unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/pagination.rs b/src/client/pagination.rs index 62cf220..57387b6 100644 --- a/src/client/pagination.rs +++ b/src/client/pagination.rs @@ -96,6 +96,7 @@ impl MapResponse> for response::Continuation { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let items = self .on_response_received_actions @@ -131,6 +132,7 @@ impl MapResponse> for response::MusicContinuation { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let mut mapper = MusicListMapper::new(lang); let mut continuations = Vec::new(); @@ -353,7 +355,7 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response("", Language::En, None).unwrap(); + items.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -375,7 +377,7 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response("", Language::En, None).unwrap(); + items.map_response("", Language::En, None, None).unwrap(); let paginator: Paginator = map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); @@ -398,7 +400,7 @@ mod tests { let items: response::Continuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response("", Language::En, None).unwrap(); + items.map_response("", Language::En, None, None).unwrap(); let paginator: Paginator = map_yt_paginator(map_res.c, None, ContinuationEndpoint::Browse); @@ -421,7 +423,7 @@ mod tests { let items: response::MusicContinuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response("", Language::En, None).unwrap(); + items.map_response("", Language::En, None, None).unwrap(); let paginator: Paginator = map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); @@ -442,7 +444,7 @@ mod tests { let items: response::MusicContinuation = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res: MapResult> = - items.map_response("", Language::En, None).unwrap(); + items.map_response("", Language::En, None, None).unwrap(); let paginator: Paginator = map_ytm_paginator(map_res.c, None, ContinuationEndpoint::MusicBrowse); diff --git a/src/client/player.rs b/src/client/player.rs index cda8f70..4061aac 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -143,6 +143,7 @@ impl MapResponse for response::Player { id: &str, _lang: Language, deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result, ExtractionError> { let deobf = Deobfuscator::new(deobf.unwrap())?; let mut warnings = vec![]; @@ -372,7 +373,10 @@ impl MapResponse for response::Player { hls_manifest_url: streaming_data.hls_manifest_url, dash_manifest_url: streaming_data.dash_manifest_url, preview_frames, - visitor_data: self.response_context.visitor_data, + visitor_data: self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)), }, warnings, }) @@ -717,7 +721,7 @@ mod tests { let resp: response::Player = serde_json::from_reader(BufReader::new(json_file)).unwrap(); let map_res = resp - .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA)) + .map_response("pPvd8UxmSbQ", Language::En, Some(&DEOBF_DATA), None) .unwrap(); assert!( diff --git a/src/client/playlist.rs b/src/client/playlist.rs index 18340df..bfdc03e 100644 --- a/src/client/playlist.rs +++ b/src/client/playlist.rs @@ -37,6 +37,7 @@ impl MapResponse for response::Playlist { id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result, ExtractionError> { let (Some(contents), Some(header)) = (self.contents, self.header) else { return Err(response::alerts_to_err(id, self.alerts)); @@ -152,7 +153,10 @@ impl MapResponse for response::Playlist { channel, last_update, last_update_txt, - visitor_data: self.response_context.visitor_data, + visitor_data: self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)), }, warnings: mapper.warnings, }) @@ -181,7 +185,7 @@ mod tests { let playlist: response::Playlist = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = playlist.map_response(id, Language::En, None).unwrap(); + let map_res = playlist.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/search.rs b/src/client/search.rs index 1701de9..1d9d186 100644 --- a/src/client/search.rs +++ b/src/client/search.rs @@ -92,6 +92,7 @@ impl MapResponse for response::Search { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result, ExtractionError> { let items = self .contents @@ -113,7 +114,10 @@ impl MapResponse for response::Search { crate::model::paginator::ContinuationEndpoint::Search, ), corrected_query: mapper.corrected_query, - visitor_data: self.response_context.visitor_data, + visitor_data: self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)), }, warnings: mapper.warnings, }) @@ -145,7 +149,8 @@ mod tests { let json_file = File::open(json_path).unwrap(); let search: response::Search = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult = search.map_response("", Language::En, None).unwrap(); + let map_res: MapResult = + search.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/trends.rs b/src/client/trends.rs index cc7408f..6e3505a 100644 --- a/src/client/trends.rs +++ b/src/client/trends.rs @@ -54,6 +54,7 @@ impl MapResponse> for response::Startpage { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result>, ExtractionError> { let grid = self .contents @@ -70,7 +71,9 @@ impl MapResponse> for response::Startpage { Ok(map_startpage_videos( grid, lang, - self.response_context.visitor_data, + self.response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)), )) } } @@ -81,6 +84,7 @@ impl MapResponse> for response::Trending { _id: &str, lang: crate::param::Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let items = self .contents @@ -146,8 +150,9 @@ mod tests { let startpage: response::Startpage = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - startpage.map_response("", Language::En, None).unwrap(); + let map_res: MapResult> = startpage + .map_response("", Language::En, None, None) + .unwrap(); assert!( map_res.warnings.is_empty(), @@ -169,8 +174,9 @@ mod tests { let startpage: response::Trending = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res: MapResult> = - startpage.map_response("", Language::En, None).unwrap(); + let map_res: MapResult> = startpage + .map_response("", Language::En, None, None) + .unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/client/url_resolver.rs b/src/client/url_resolver.rs index 6b9e83b..6ca8cb6 100644 --- a/src/client/url_resolver.rs +++ b/src/client/url_resolver.rs @@ -328,6 +328,7 @@ impl MapResponse for response::ResolvedUrl { _id: &str, _lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result, ExtractionError> { let pt = self.endpoint.page_type(); if let NavigationEndpoint::Browse { diff --git a/src/client/video_details.rs b/src/client/video_details.rs index b04e806..92570cc 100644 --- a/src/client/video_details.rs +++ b/src/client/video_details.rs @@ -82,6 +82,7 @@ impl MapResponse for response::VideoDetails { id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + vdata: Option<&str>, ) -> Result, ExtractionError> { let mut warnings = Vec::new(); @@ -256,7 +257,10 @@ impl MapResponse for response::VideoDetails { _ => return Err(ExtractionError::InvalidData("invalid channel link".into())), }; - let visitor_data = self.response_context.visitor_data; + let visitor_data = self + .response_context + .visitor_data + .or_else(|| vdata.map(str::to_owned)); let recommended = contents .two_column_watch_next_results .secondary_results @@ -369,6 +373,7 @@ impl MapResponse> for response::VideoComments { _id: &str, lang: Language, _deobf: Option<&crate::deobfuscate::DeobfData>, + _vdata: Option<&str>, ) -> Result>, ExtractionError> { let received_endpoints = self.on_response_received_endpoints; let mut warnings = received_endpoints.warnings; @@ -561,7 +566,7 @@ mod tests { let details: response::VideoDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = details.map_response(id, Language::En, None).unwrap(); + let map_res = details.map_response(id, Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), @@ -581,7 +586,9 @@ mod tests { let details: response::VideoDetails = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let err = details.map_response("", Language::En, None).unwrap_err(); + let err = details + .map_response("", Language::En, None, None) + .unwrap_err(); assert!(matches!( err, crate::error::ExtractionError::NotFound { .. } @@ -597,7 +604,7 @@ mod tests { let comments: response::VideoComments = serde_json::from_reader(BufReader::new(json_file)).unwrap(); - let map_res = comments.map_response("", Language::En, None).unwrap(); + let map_res = comments.map_response("", Language::En, None, None).unwrap(); assert!( map_res.warnings.is_empty(), diff --git a/src/util/dictionary.rs b/src/util/dictionary.rs index 83d6c51..3caffd9 100644 --- a/src/util/dictionary.rs +++ b/src/util/dictionary.rs @@ -1324,27 +1324,36 @@ pub(crate) fn entry(lang: Language) -> Entry { }, Language::En | Language::EnGb | Language::EnIn => Entry { timeago_tokens: ::phf::Map { - key: 7485420634051515786, + key: 12913932095322966823, disps: &[ - (0, 7), - (8, 12), - (5, 0), + (2, 0), + (2, 0), + (5, 18), + (4, 15), + (2, 17), ], entries: &[ - ("months", TaToken { n: 1, unit: Some(TimeUnit::Month) }), - ("month", TaToken { n: 1, unit: Some(TimeUnit::Month) }), - ("hours", TaToken { n: 1, unit: Some(TimeUnit::Hour) }), - ("days", TaToken { n: 1, unit: Some(TimeUnit::Day) }), - ("years", TaToken { n: 1, unit: Some(TimeUnit::Year) }), - ("week", TaToken { n: 1, unit: Some(TimeUnit::Week) }), - ("second", TaToken { n: 1, unit: Some(TimeUnit::Second) }), ("day", TaToken { n: 1, unit: Some(TimeUnit::Day) }), + ("month", TaToken { n: 1, unit: Some(TimeUnit::Month) }), + ("minutes", TaToken { n: 1, unit: Some(TimeUnit::Minute) }), + ("min", TaToken { n: 1, unit: Some(TimeUnit::Minute) }), + ("y", TaToken { n: 1, unit: Some(TimeUnit::Year) }), + ("days", TaToken { n: 1, unit: Some(TimeUnit::Day) }), ("minute", TaToken { n: 1, unit: Some(TimeUnit::Minute) }), ("seconds", TaToken { n: 1, unit: Some(TimeUnit::Second) }), + ("s", TaToken { n: 1, unit: Some(TimeUnit::Second) }), + ("mo", TaToken { n: 1, unit: Some(TimeUnit::Month) }), + ("hours", TaToken { n: 1, unit: Some(TimeUnit::Hour) }), + ("w", TaToken { n: 1, unit: Some(TimeUnit::Week) }), + ("h", TaToken { n: 1, unit: Some(TimeUnit::Hour) }), + ("d", TaToken { n: 1, unit: Some(TimeUnit::Day) }), + ("second", TaToken { n: 1, unit: Some(TimeUnit::Second) }), + ("years", TaToken { n: 1, unit: Some(TimeUnit::Year) }), + ("months", TaToken { n: 1, unit: Some(TimeUnit::Month) }), + ("week", TaToken { n: 1, unit: Some(TimeUnit::Week) }), ("weeks", TaToken { n: 1, unit: Some(TimeUnit::Week) }), - ("hour", TaToken { n: 1, unit: Some(TimeUnit::Hour) }), - ("minutes", TaToken { n: 1, unit: Some(TimeUnit::Minute) }), ("year", TaToken { n: 1, unit: Some(TimeUnit::Year) }), + ("hour", TaToken { n: 1, unit: Some(TimeUnit::Hour) }), ], }, date_order: &[DateCmp::D, DateCmp::Y], diff --git a/testfiles/dict/dictionary.json b/testfiles/dict/dictionary.json index 6bca334..d58d03a 100644 --- a/testfiles/dict/dictionary.json +++ b/testfiles/dict/dictionary.json @@ -754,7 +754,14 @@ "week": "W", "weeks": "W", "year": "Y", - "years": "Y" + "years": "Y", + "y": "Y", + "mo": "M", + "w": "W", + "d": "D", + "h": "h", + "min": "m", + "s": "s" }, "date_order": "DY", "months": { diff --git a/tests/youtube.rs b/tests/youtube.rs index bd5e1e3..3aacb2f 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -2336,7 +2336,12 @@ fn lang() -> Language { /// Get a new RustyPipe instance #[fixture] fn rp(lang: Language) -> RustyPipe { - RustyPipe::builder().strict().lang(lang).build() + let vdata = std::env::var("YT_VDATA").ok(); + RustyPipe::builder() + .strict() + .lang(lang) + .visitor_data_opt(vdata) + .build() } /// Get a flag signaling if the language is set to English