Sulkta-Coop/rustypipe
Archived
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add visitor data parameter

Browse source
This commit is contained in:
ThetaDev 2022-10-24 07:45:57 +02:00
parent 39b32da5a4
commit 9d0ae0e9c2
13 changed files with 113 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/src/main.rs
View file

@ -145,7 +145,7 @@ async fn download_playlist(
let mut playlist = rp.query().playlist(id).await.unwrap();
playlist
.videos
.extend_pages(rp.query(), usize::MAX)
.extend_pages(&rp.query(), usize::MAX)
.await
.unwrap();

16
codegen/src/download_testfiles.rs
View file

@ -158,7 +158,7 @@ async fn playlist_cont(testfiles: &Path) {
.unwrap();
let rp = rp_testfile(&json_path);
playlist.videos.next(rp.query()).await.unwrap().unwrap();
playlist.videos.next(&rp.query()).await.unwrap().unwrap();
}
async fn video_details(testfiles: &Path) {
@ -196,7 +196,7 @@ async fn comments_top(testfiles: &Path) {
let rp = rp_testfile(&json_path);
details
.top_comments
.next(rp.query())
.next(&rp.query())
.await
.unwrap()
.unwrap();
@ -216,7 +216,7 @@ async fn comments_latest(testfiles: &Path) {
let rp = rp_testfile(&json_path);
details
.latest_comments
.next(rp.query())
.next(&rp.query())
.await
.unwrap()
.unwrap();
@ -234,7 +234,7 @@ async fn recommendations(testfiles: &Path) {
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let rp = rp_testfile(&json_path);
details.recommended.next(rp.query()).await.unwrap();
details.recommended.next(&rp.query()).await.unwrap();
}
async fn channel_videos(testfiles: &Path) {
@ -304,7 +304,7 @@ async fn channel_videos_cont(testfiles: &Path) {
.unwrap();
let rp = rp_testfile(&json_path);
videos.content.next(rp.query()).await.unwrap().unwrap();
videos.content.next(&rp.query()).await.unwrap().unwrap();
}
async fn channel_playlists_cont(testfiles: &Path) {
@ -323,7 +323,7 @@ async fn channel_playlists_cont(testfiles: &Path) {
.unwrap();
let rp = rp_testfile(&json_path);
playlists.content.next(rp.query()).await.unwrap().unwrap();
playlists.content.next(&rp.query()).await.unwrap().unwrap();
}
async fn search(testfiles: &Path) {
@ -350,7 +350,7 @@ async fn search_cont(testfiles: &Path) {
let search = rp.query().search("doobydoobap").await.unwrap();
let rp = rp_testfile(&json_path);
search.items.next(rp.query()).await.unwrap().unwrap();
search.items.next(&rp.query()).await.unwrap().unwrap();
}
async fn search_playlists(testfiles: &Path) {
@ -412,7 +412,7 @@ async fn startpage_cont(testfiles: &Path) {
let startpage = rp.query().startpage().await.unwrap();
let rp = rp_testfile(&json_path);
startpage.next(rp.query()).await.unwrap();
startpage.next(&rp.query()).await.unwrap();
}
async fn trending(testfiles: &Path) {

12
src/client/channel.rs
View file

@ -39,7 +39,7 @@ enum Params {
impl RustyPipeQuery {
pub async fn channel_videos(
self,
&self,
channel_id: &str,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
self.channel_videos_ordered(channel_id, ChannelOrder::default())
@ -47,11 +47,11 @@ impl RustyPipeQuery {
}
pub async fn channel_videos_ordered(
self,
&self,
channel_id: &str,
order: ChannelOrder,
) -> Result<Channel<Paginator<VideoItem>>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QChannel {
context,
browse_id: channel_id,
@ -73,10 +73,10 @@ impl RustyPipeQuery {
}
pub async fn channel_playlists(
self,
&self,
channel_id: &str,
) -> Result<Channel<Paginator<PlaylistItem>>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QChannel {
context,
browse_id: channel_id,
@ -94,7 +94,7 @@ impl RustyPipeQuery {
}
pub async fn channel_info(&self, channel_id: &str) -> Result<Channel<ChannelInfo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QChannel {
context,
browse_id: channel_id,

2
src/client/channel_rss.rs
View file

@ -9,7 +9,7 @@ use crate::{
use super::{response, RustyPipeQuery};
impl RustyPipeQuery {
pub async fn channel_rss(self, channel_id: &str) -> Result<ChannelRss, Error> {
pub async fn channel_rss(&self, channel_id: &str) -> Result<ChannelRss, Error> {
let url = format!(
"https://www.youtube.com/feeds/videos.xml?channel_id={}",
channel_id

49
src/client/mod.rs
View file

@ -45,7 +45,7 @@ use crate::{
/// - **Desktop**: used by youtube.com
/// - **DesktopMusic**: used by music.youtube.com, can access special music data,
/// cannot access non-music content
/// - **TvHtml5Embed**: (probably) used by Smart TVs, can access age-restricted videos
/// - **TvHtml5Embed**: used by Smart TVs, can access age-restricted videos
/// - **Android**: used by the Android app, no obfuscated URLs, includes lower resolution audio streams
/// - **Ios**: used by the iOS app, no obfuscated URLs
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
@ -69,7 +69,7 @@ impl ClientType {
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct YTContext<'a> {
pub struct YTContext<'a> {
client: ClientInfo<'a>,
/// only used on desktop
#[serde(skip_serializing_if = "Option::is_none")]
@ -93,7 +93,7 @@ struct ClientInfo<'a> {
#[serde(skip_serializing_if = "Option::is_none")]
original_url: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
visitor_data: Option<String>,
visitor_data: Option<&'a str>,
hl: Language,
gl: Country,
}
@ -183,6 +183,7 @@ struct RustyPipeRef {
reporter: Option<Box<dyn Reporter>>,
n_http_retries: u32,
consent_cookie: String,
visitor_data: Option<String>,
cache: CacheHolder,
default_opts: RustyPipeOpts,
}
@ -200,6 +201,7 @@ pub struct RustyPipeBuilder {
reporter: Option<Box<dyn Reporter>>,
n_http_retries: u32,
user_agent: String,
visitor_data: Option<String>,
default_opts: RustyPipeOpts,
}
@ -292,6 +294,7 @@ impl RustyPipeBuilder {
reporter: Some(Box::new(FileReporter::default())),
n_http_retries: 2,
user_agent: DEFAULT_UA.to_owned(),
visitor_data: None,
}
}
@ -332,6 +335,7 @@ impl RustyPipeBuilder {
CONSENT_COOKIE_YES,
rand::thread_rng().gen_range(100..1000)
),
visitor_data: self.visitor_data,
cache: CacheHolder {
desktop_client: RwLock::new(cdata.desktop_client),
music_client: RwLock::new(cdata.music_client),
@ -434,6 +438,11 @@ impl RustyPipeBuilder {
self.default_opts.strict = true;
self
}
pub fn visitor_data(mut self, visitor_data: &str) -> Self {
self.visitor_data = Some(visitor_data.to_owned());
self
}
}
impl Default for RustyPipe {
@ -745,7 +754,12 @@ impl RustyPipeQuery {
/// # Parameters
/// - `ctype`: Client type (`Desktop`, `DesktopMusic`, `Android`, ...)
/// - `localized`: Whether to include the configured language and country
async fn get_context(&self, ctype: ClientType, localized: bool) -> YTContext {
pub async fn get_context<'a>(
&'a self,
ctype: ClientType,
localized: bool,
visitor_data: Option<&'a str>,
) -> YTContext {
let hl = match localized {
true => self.opts.lang,
false => Language::En,
@ -754,6 +768,7 @@ impl RustyPipeQuery {
true => self.opts.country,
false => Country::Us,
};
let visitor_data = self.client.inner.visitor_data.as_deref().or(visitor_data);
match ctype {
ClientType::Desktop => YTContext {
@ -764,7 +779,7 @@ impl RustyPipeQuery {
device_model: None,
platform: "DESKTOP",
original_url: Some("https://www.youtube.com/"),
visitor_data: None,
visitor_data,
hl,
gl,
},
@ -780,7 +795,7 @@ impl RustyPipeQuery {
device_model: None,
platform: "DESKTOP",
original_url: Some("https://music.youtube.com/"),
visitor_data: None,
visitor_data,
hl,
gl,
},
@ -796,7 +811,7 @@ impl RustyPipeQuery {
device_model: None,
platform: "TV",
original_url: None,
visitor_data: None,
visitor_data,
hl,
gl,
},
@ -814,7 +829,7 @@ impl RustyPipeQuery {
device_model: None,
platform: "MOBILE",
original_url: None,
visitor_data: None,
visitor_data,
hl,
gl,
},
@ -830,7 +845,7 @@ impl RustyPipeQuery {
device_model: Some(IOS_DEVICE_MODEL),
platform: "MOBILE",
original_url: None,
visitor_data: None,
visitor_data,
hl,
gl,
},
@ -1085,6 +1100,22 @@ impl RustyPipeQuery {
self.execute_request_deobf::<R, M, B>(ctype, operation, id, endpoint, body, None)
.await
}
/// Execute a request to the YouTube API and return the response string
pub async fn raw<B: Serialize + ?Sized>(
&self,
ctype: ClientType,
endpoint: &str,
body: &B,
) -> Result<String, Error> {
let request = self
.request_builder(ctype, endpoint)
.await
.json(body)
.build()?;
self.client.http_request_txt(request).await
}
}
/// Implement this for YouTube API response structs that need to be mapped to

33
src/client/pagination.rs
View file

@ -10,13 +10,14 @@ use super::{response, ClientType, MapResponse, QContinuation, RustyPipeQuery};
impl RustyPipeQuery {
pub async fn continuation<T: TryFrom<YouTubeItem>>(
self,
&self,
ctoken: &str,
endpoint: ContinuationEndpoint,
visitor_data: Option<&str>,
) -> Result<Paginator<T>, Error> {
let mut context = self.get_context(ClientType::Desktop, true).await;
context.client.visitor_data = visitor_data.map(str::to_owned);
let context = self
.get_context(ClientType::Desktop, true, visitor_data)
.await;
let request_body = QContinuation {
context,
continuation: ctoken,
@ -87,7 +88,7 @@ impl<T: TryFrom<YouTubeItem>> MapResponse<Paginator<T>> for response::Continuati
}
impl<T: TryFrom<YouTubeItem>> Paginator<T> {
pub async fn next(&self, query: RustyPipeQuery) -> Result<Option<Self>, Error> {
pub async fn next(&self, query: &RustyPipeQuery) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(
query
@ -98,7 +99,7 @@ impl<T: TryFrom<YouTubeItem>> Paginator<T> {
})
}
pub async fn extend(&mut self, query: RustyPipeQuery) -> Result<bool, Error> {
pub async fn extend(&mut self, query: &RustyPipeQuery) -> Result<bool, Error> {
match self.next(query).await {
Ok(Some(paginator)) => {
let mut items = paginator.items;
@ -113,11 +114,11 @@ impl<T: TryFrom<YouTubeItem>> Paginator<T> {
pub async fn extend_pages(
&mut self,
query: RustyPipeQuery,
query: &RustyPipeQuery,
n_pages: usize,
) -> Result<(), Error> {
for _ in 0..n_pages {
match self.extend(query.clone()).await {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
@ -128,11 +129,11 @@ impl<T: TryFrom<YouTubeItem>> Paginator<T> {
pub async fn extend_limit(
&mut self,
query: RustyPipeQuery,
query: &RustyPipeQuery,
n_items: usize,
) -> Result<(), Error> {
while self.items.len() < n_items {
match self.extend(query.clone()).await {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
@ -143,7 +144,7 @@ impl<T: TryFrom<YouTubeItem>> Paginator<T> {
}
impl Paginator<Comment> {
pub async fn next(&self, query: RustyPipeQuery) -> Result<Option<Self>, Error> {
pub async fn next(&self, query: &RustyPipeQuery) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(
query
@ -156,7 +157,7 @@ impl Paginator<Comment> {
}
impl Paginator<PlaylistVideo> {
pub async fn next(&self, query: RustyPipeQuery) -> Result<Option<Self>, Error> {
pub async fn next(&self, query: &RustyPipeQuery) -> Result<Option<Self>, Error> {
Ok(match &self.ctoken {
Some(ctoken) => Some(query.playlist_continuation(ctoken).await?),
None => None,
@ -167,7 +168,7 @@ impl Paginator<PlaylistVideo> {
macro_rules! paginator {
($entity_type:ty) => {
impl Paginator<$entity_type> {
pub async fn extend(&mut self, query: RustyPipeQuery) -> Result<bool, Error> {
pub async fn extend(&mut self, query: &RustyPipeQuery) -> Result<bool, Error> {
match self.next(query).await {
Ok(Some(paginator)) => {
let mut items = paginator.items;
@ -182,11 +183,11 @@ macro_rules! paginator {
pub async fn extend_pages(
&mut self,
query: RustyPipeQuery,
query: &RustyPipeQuery,
n_pages: usize,
) -> Result<(), Error> {
for _ in 0..n_pages {
match self.extend(query.clone()).await {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}
@ -197,11 +198,11 @@ macro_rules! paginator {
pub async fn extend_limit(
&mut self,
query: RustyPipeQuery,
query: &RustyPipeQuery,
n_items: usize,
) -> Result<(), Error> {
while self.items.len() < n_items {
match self.extend(query.clone()).await {
match self.extend(query).await {
Ok(false) => break,
Err(e) => return Err(e),
_ => {}

6
src/client/player.rs
View file

@ -58,7 +58,7 @@ struct QContentPlaybackContext {
}
impl RustyPipeQuery {
pub async fn player(self, video_id: &str) -> Result<VideoPlayer, Error> {
pub async fn player(&self, video_id: &str) -> Result<VideoPlayer, Error> {
let q1 = self.clone();
let android_res = q1.player_from_client(video_id, ClientType::Android).await;
@ -75,12 +75,12 @@ impl RustyPipeQuery {
}
pub async fn player_from_client(
self,
&self,
video_id: &str,
client_type: ClientType,
) -> Result<VideoPlayer, Error> {
let (context, deobf) = tokio::join!(
self.get_context(client_type, false),
self.get_context(client_type, false, None),
self.client.get_deobf()
);
let deobf = deobf?;

8
src/client/playlist.rs
View file

@ -14,8 +14,8 @@ use crate::{
use super::{response, ClientType, MapResponse, MapResult, QBrowse, QContinuation, RustyPipeQuery};
impl RustyPipeQuery {
pub async fn playlist(self, playlist_id: &str) -> Result<Playlist, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
pub async fn playlist(&self, playlist_id: &str) -> Result<Playlist, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QBrowse {
context,
browse_id: "VL".to_owned() + playlist_id,
@ -32,10 +32,10 @@ impl RustyPipeQuery {
}
pub async fn playlist_continuation(
self,
&self,
ctoken: &str,
) -> Result<Paginator<PlaylistVideo>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QContinuation {
context,
continuation: ctoken,

10
src/client/search.rs
View file

@ -21,8 +21,8 @@ struct QSearch<'a> {
}
impl RustyPipeQuery {
pub async fn search(self, query: &str) -> Result<SearchResult, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
pub async fn search(&self, query: &str) -> Result<SearchResult, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QSearch {
context,
query,
@ -40,11 +40,11 @@ impl RustyPipeQuery {
}
pub async fn search_filter(
self,
&self,
query: &str,
filter: &SearchFilter,
) -> Result<SearchResult, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QSearch {
context,
query,
@ -61,7 +61,7 @@ impl RustyPipeQuery {
.await
}
pub async fn search_suggestion(self, query: &str) -> Result<Vec<String>, Error> {
pub async fn search_suggestion(&self, query: &str) -> Result<Vec<String>, Error> {
let url = url::Url::parse_with_params("https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&gs_rn=64&gs_ri=youtube&ds=yt&cp=1&gs_id=4&xhr=t&xssi=t",
&[("hl", self.opts.lang.to_string()), ("gl", self.opts.country.to_string()), ("q", query.to_string())]
).map_err(|_| Error::Other("could not build url".into()))?;

8
src/client/trends.rs
View file

@ -11,8 +11,8 @@ use crate::{
use super::{response, ClientType, MapResponse, QBrowse, RustyPipeQuery};
impl RustyPipeQuery {
pub async fn startpage(self) -> Result<Paginator<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
pub async fn startpage(&self) -> Result<Paginator<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QBrowse {
context,
browse_id: "FEwhat_to_watch".to_owned(),
@ -28,8 +28,8 @@ impl RustyPipeQuery {
.await
}
pub async fn trending(self) -> Result<Vec<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
pub async fn trending(&self) -> Result<Vec<VideoItem>, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QBrowse {
context,
browse_id: "FEtrending".to_owned(),

2
src/client/url_resolver.rs
View file

@ -165,7 +165,7 @@ impl RustyPipeQuery {
}
async fn _navigation_resolve_url(&self, url_path: &str) -> Result<UrlTarget, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QResolveUrl {
context,
url: format!("https://www.youtube.com{}", url_path),

11
src/client/video_details.rs
View file

@ -28,8 +28,8 @@ struct QVideo<'a> {
}
impl RustyPipeQuery {
pub async fn video_details(self, video_id: &str) -> Result<VideoDetails, Error> {
let context = self.get_context(ClientType::Desktop, true).await;
pub async fn video_details(&self, video_id: &str) -> Result<VideoDetails, Error> {
let context = self.get_context(ClientType::Desktop, true, None).await;
let request_body = QVideo {
context,
video_id,
@ -48,12 +48,13 @@ impl RustyPipeQuery {
}
pub async fn video_comments(
self,
&self,
ctoken: &str,
visitor_data: Option<&str>,
) -> Result<Paginator<Comment>, Error> {
let mut context = self.get_context(ClientType::Desktop, true).await;
context.client.visitor_data = visitor_data.map(str::to_owned);
let context = self
.get_context(ClientType::Desktop, true, visitor_data)
.await;
let request_body = QContinuation {
context,
continuation: ctoken,

25
tests/youtube.rs
View file

@ -338,7 +338,7 @@ async fn playlist_cont() {
playlist
.videos
.extend_pages(rp.query(), usize::MAX)
.extend_pages(&rp.query(), usize::MAX)
.await
.unwrap();
assert!(playlist.videos.items.len() > 100);
@ -354,7 +354,11 @@ async fn playlist_cont2() {
.await
.unwrap();
playlist.videos.extend_limit(rp.query(), 101).await.unwrap();
playlist
.videos
.extend_limit(&rp.query(), 101)
.await
.unwrap();
assert!(playlist.videos.items.len() > 100);
assert!(playlist.videos.count.unwrap() > 100);
}
@ -797,7 +801,12 @@ async fn get_video_details_not_found() {
async fn get_video_recommendations() {
let rp = RustyPipe::builder().strict().build();
let details = rp.query().video_details("ZeerrnuLi5E").await.unwrap();
let next_recommendations = details.recommended.next(rp.query()).await.unwrap().unwrap();
let next_recommendations = details
.recommended
.next(&rp.query())
.await
.unwrap()
.unwrap();
// dbg!(&next_recommendations);
assert!(
@ -815,7 +824,7 @@ async fn get_video_comments() {
let top_comments = details
.top_comments
.next(rp.query())
.next(&rp.query())
.await
.unwrap()
.unwrap();
@ -837,7 +846,7 @@ async fn get_video_comments() {
let latest_comments = details
.latest_comments
.next(rp.query())
.next(&rp.query())
.await
.unwrap()
.unwrap();
@ -892,7 +901,7 @@ async fn channel_videos(#[case] order: ChannelOrder) {
_ => unimplemented!(),
}
let next = channel.content.next(rp.query()).await.unwrap().unwrap();
let next = channel.content.next(&rp.query()).await.unwrap().unwrap();
assert!(
!next.is_exhausted() && !next.items.is_empty(),
"no more videos"
@ -915,7 +924,7 @@ async fn channel_playlists() {
"got no playlists"
);
let next = channel.content.next(rp.query()).await.unwrap().unwrap();
let next = channel.content.next(&rp.query()).await.unwrap().unwrap();
assert!(
!next.is_exhausted() && !next.items.is_empty(),
"no more playlists"
@ -1267,7 +1276,7 @@ async fn startpage_cont() {
let rp = RustyPipe::builder().strict().build();
let startpage = rp.query().startpage().await.unwrap();
let next = startpage.next(rp.query()).await.unwrap().unwrap();
let next = startpage.next(&rp.query()).await.unwrap().unwrap();
assert!(
next.items.len() >= 20,