diff --git a/.forgejo/workflows/ci.yaml b/.forgejo/workflows/ci.yaml index 8622f8b..60230e6 100644 --- a/.forgejo/workflows/ci.yaml +++ b/.forgejo/workflows/ci.yaml @@ -28,7 +28,9 @@ jobs: run: cargo clippy --all --tests --features=rss,indicatif,audiotag -- -D warnings - name: 🧪 Test - run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace + run: | + echo "${{ secrets.RUSTYPIPE_CACHE }}" > rustypipe_cache.json + cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace env: ALL_PROXY: "http://warpproxy:8124" diff --git a/cli/src/main.rs b/cli/src/main.rs index b48bca3..eb29cdf 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -191,6 +191,8 @@ enum Commands { }, /// Get a YouTube visitor data cookie Vdata, + /// Log in using your Google account + Login, } #[derive(Default, Copy, Clone, ValueEnum)] @@ -1208,6 +1210,22 @@ async fn run() -> anyhow::Result<()> { let vd = rp.query().get_visitor_data().await?; println!("{vd}"); } + Commands::Login => { + match rp.user_auth_check_login().await { + Ok(_) => {} + Err(rustypipe::error::Error::Auth(_)) => { + let device_code = rp.user_auth_get_code().await?; + println!( + "Open {} and enter the following code:", + device_code.verification_url + ); + anstream::println!("{}", device_code.user_code.blue()); + rp.user_auth_wait_for_login(&device_code).await?; + } + Err(e) => return Err(e.into()), + } + anstream::println!("{}", "Logged in.".green()); + } }; Ok(()) } diff --git a/src/client/mod.rs b/src/client/mod.rs index a1a707b..4dee6f7 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -24,7 +24,7 @@ mod channel_rss; use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use std::{borrow::Cow, fmt::Debug, time::Duration}; use once_cell::sync::Lazy; @@ -32,8 +32,9 @@ use regex::Regex; use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use time::OffsetDateTime; -use tokio::sync::RwLock; +use tokio::sync::RwLock as AsyncRwLock; +use crate::error::AuthError; use crate::{ cache::{CacheStorage, FileStorage, DEFAULT_CACHE_FILE}, deobfuscate::DeobfData, @@ -198,6 +199,87 @@ struct QContinuation<'a> { continuation: &'a str, } +#[derive(Debug, Serialize)] +struct OauthCodeRequest { + client_id: &'static str, + device_id: String, + device_model: &'static str, + scope: &'static str, +} + +/// Device code used for logging a user into YouTube +/// +/// The login process works as follows: +/// 1. Obtain a user code and show it to the user +/// 2. The user opens the login page under , enters the code and logs in with his account +/// 3. The application has to check periodically if the login has succeeded using [`RustyPipe::oauth_login`] or [`RustyPipe::oauth_wait_for_login`] +/// 4. If the login is successful, the application receives a valid access/refresh token pair which can be used to access YouTube +#[derive(Debug, Deserialize)] +pub struct OauthDeviceCode { + device_code: String, + /// Code to be shown to the user to log himself in + pub user_code: String, + /// Time in seconds until the code expires + pub expires_in: u32, + /// Interval in seconds for checking if the login was completed + pub interval: u32, + /// URL to the login page () + pub verification_url: String, +} + +#[derive(Debug, Serialize)] +struct OauthTokenRequest<'a> { + client_id: &'static str, + client_secret: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + code: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + refresh_token: Option<&'a str>, + grant_type: &'static str, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum OauthTokenResponse { + Ok(OauthTokenResponseInner), + Error { + error: String, + #[serde(default)] + error_description: String, + }, +} + +#[derive(Debug, Deserialize)] +struct OauthTokenResponseInner { + access_token: String, + refresh_token: Option, + expires_in: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct OauthToken { + access_token: String, + refresh_token: String, + #[serde(with = "time::serde::rfc3339")] + expires_at: OffsetDateTime, +} + +impl OauthToken { + fn from_response( + value: OauthTokenResponseInner, + refresh_token: Option, + ) -> Result { + Ok(Self { + access_token: value.access_token, + refresh_token: value + .refresh_token + .or(refresh_token) + .ok_or(Error::Other("missing refresh token".into()))?, + expires_at: util::now_sec() + Duration::from_secs(value.expires_in.into()), + }) + } +} + const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0"; const MOBILE_UA: &str = "Mozilla/5.0 (Android 14; Mobile; rv:129.0) Gecko/129.0 Firefox/129.0"; const TV_UA: &str = "Mozilla/5.0 (SMART-TV; Linux; Tizen 5.0) AppleWebKit/538.1 (KHTML, like Gecko) Version/5.0 NativeTVAds Safari/538.1"; @@ -225,6 +307,11 @@ const TV_CLIENT_VERSION: &str = "7.20241008.14.02"; const APP_CLIENT_VERSION: &str = "18.03.33"; const IOS_DEVICE_MODEL: &str = "iPhone14,5"; +const OAUTH_CLIENT_ID: &str = + "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com"; +const OAUTH_CLIENT_SECRET: &str = "SboVhoG9s0rNafixCSGGKXAT"; +const OAUTH_SCOPES: &str = "http://gdata.youtube.com https://www.googleapis.com/auth/youtube"; + static CLIENT_VERSION_REGEX: Lazy = Lazy::new(|| Regex::new(r#""INNERTUBE_CONTEXT_CLIENT_VERSION":"([\w\d\._-]+?)""#).unwrap()); static VISITOR_DATA_REGEX: Lazy = @@ -263,6 +350,7 @@ struct RustyPipeOpts { country: Country, report: bool, strict: bool, + auth: Option, visitor_data: Option, } @@ -377,6 +465,7 @@ impl Default for RustyPipeOpts { country: Country::Us, report: false, strict: false, + auth: None, visitor_data: None, } } @@ -384,8 +473,9 @@ impl Default for RustyPipeOpts { #[derive(Default, Debug)] struct CacheHolder { - clients: HashMap>>, - deobf: RwLock>, + clients: HashMap>>, + deobf: AsyncRwLock>, + oauth_token: RwLock>, } #[derive(Default, Debug, Clone, Serialize, Deserialize)] @@ -394,6 +484,8 @@ struct CacheData { clients: HashMap>, #[serde(skip_serializing_if = "CacheEntry::is_none")] deobf: CacheEntry, + #[serde(skip_serializing_if = "Option::is_none")] + oauth_token: Option, } #[derive(Default, Debug, Clone, Serialize, Deserialize)] @@ -532,7 +624,12 @@ impl RustyPipeBuilder { ClientType::Tv, ] .into_iter() - .map(|c| (c, RwLock::new(cdata.clients.remove(&c).unwrap_or_default()))) + .map(|c| { + ( + c, + AsyncRwLock::new(cdata.clients.remove(&c).unwrap_or_default()), + ) + }) .collect::>(); Ok(RustyPipe { @@ -547,7 +644,8 @@ impl RustyPipeBuilder { n_http_retries: self.n_http_retries, cache: CacheHolder { clients: cache_clients, - deobf: RwLock::new(cdata.deobf), + deobf: AsyncRwLock::new(cdata.deobf), + oauth_token: RwLock::new(cdata.oauth_token), }, default_opts: self.default_opts, user_agent, @@ -693,6 +791,20 @@ impl RustyPipeBuilder { self } + /// Enable authentication for all requests + #[must_use] + pub fn authenticated(mut self) -> Self { + self.default_opts.auth = Some(true); + self + } + + /// Disable authentication for all requests + #[must_use] + pub fn unauthenticated(mut self) -> Self { + self.default_opts.auth = Some(false); + self + } + /// Set the YouTube visitor data cookie /// /// YouTube assigns a session cookie to each user which is used for personalized @@ -970,6 +1082,7 @@ impl RustyPipe { let cdata = CacheData { clients: cache_clients, deobf: self.inner.cache.deobf.read().await.clone(), + oauth_token: self.inner.cache.oauth_token.read().unwrap().clone(), }; match serde_json::to_string(&cdata) { @@ -1033,6 +1146,175 @@ impl RustyPipe { } } } + + /// Get a new device code for logging into YouTube + pub async fn user_auth_get_code(&self) -> Result { + tracing::debug!("getting OAuth user code"); + + let code_request = OauthCodeRequest { + client_id: OAUTH_CLIENT_ID, + device_id: util::random_uuid(), + device_model: "ytlr:samsung:smarttv", + scope: OAUTH_SCOPES, + }; + + self.inner + .http + .post("https://www.youtube.com/o/oauth2/device/code") + .header(header::USER_AGENT, TV_UA) + .header(header::ORIGIN, YOUTUBE_HOME_URL) + .header(header::REFERER, YOUTUBE_TV_URL) + .json(&code_request) + .send() + .await? + .error_for_status()? + .json::() + .await + .map_err(Error::from) + } + + /// Attempt to log in the user using the given device code + /// + /// Returns `true` if the user has successfully logged in using the code. + /// + /// Returns `false` if the user has not logged in yet, in this case repeat + /// the login attempt after a few seconds. + /// The function [`RustyPipe::oauth_wait_for_login`] does this automatically. + pub async fn user_auth_login(&self, code: &OauthDeviceCode) -> Result { + tracing::debug!("OAuth login attempt (user_code: {})", code.user_code); + + let token_request = OauthTokenRequest { + client_id: OAUTH_CLIENT_ID, + client_secret: OAUTH_CLIENT_SECRET, + code: Some(&code.device_code), + refresh_token: None, + grant_type: "http://oauth.net/grant_type/device/1.0", + }; + + let token_response = self + .inner + .http + .post("https://www.youtube.com/o/oauth2/token") + .header(header::USER_AGENT, TV_UA) + .header(header::ORIGIN, YOUTUBE_HOME_URL) + .header(header::REFERER, YOUTUBE_TV_URL) + .json(&token_request) + .send() + .await? + .error_for_status()? + .json::() + .await?; + + match token_response { + OauthTokenResponse::Ok(token) => { + let token = OauthToken::from_response(token, None)?; + { + let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); + *cache_token = Some(token); + } + self.store_cache().await; + Ok(true) + } + OauthTokenResponse::Error { + error, + error_description, + } => match error.as_str() { + "authorization_pending" => Ok(false), + "expired_token" => Err(Error::Auth(AuthError::DeviceCodeExpired)), + _ => Err(Error::Auth(AuthError::Other(format!( + "{error}: {error_description}" + )))), + }, + } + } + + /// Attempt to refresh the OAuth access token to check if the user is successfully logged in + /// and the session is still valid. + pub async fn user_auth_check_login(&self) -> Result<(), Error> { + let cache_token = self.inner.cache.oauth_token.read().unwrap().clone(); + if let Some(token) = cache_token { + let token = self.refresh_token(&token.refresh_token).await?; + { + let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); + *cache_token = Some(token.clone()); + } + self.store_cache().await; + Ok(()) + } else { + Err(Error::Auth(AuthError::NoLogin)) + } + } + + /// Attempt to log in the user using the given device code. + /// + /// This function waits until the login was successful or an error occurred. + pub async fn user_auth_wait_for_login(&self, code: &OauthDeviceCode) -> Result<(), Error> { + while !self.user_auth_login(code).await? { + tokio::time::sleep(Duration::from_secs(code.interval.into())).await; + } + Ok(()) + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + tracing::debug!("refreshing OAuth token"); + + let token_request = OauthTokenRequest { + client_id: OAUTH_CLIENT_ID, + client_secret: OAUTH_CLIENT_SECRET, + code: None, + refresh_token: Some(refresh_token), + grant_type: "refresh_token", + }; + + let token_response = self + .inner + .http + .post("https://www.youtube.com/o/oauth2/token") + .header(header::USER_AGENT, TV_UA) + .header(header::ORIGIN, YOUTUBE_HOME_URL) + .header(header::REFERER, YOUTUBE_TV_URL) + .json(&token_request) + .send() + .await? + .error_for_status()? + .json::() + .await?; + + match token_response { + OauthTokenResponse::Ok(token) => { + OauthToken::from_response(token, Some(refresh_token.to_owned())) + } + OauthTokenResponse::Error { + error, + error_description, + } => Err(Error::Auth(AuthError::Refresh(format!( + "{error}: {error_description}" + )))), + } + } + + /// Get the OAuth access token for accessing YouTube as an authenticated user + pub async fn user_auth_access_token(&self) -> Result { + let cache_token = self.inner.cache.oauth_token.read().unwrap().clone(); + if let Some(token) = cache_token { + if token.expires_at < (OffsetDateTime::now_utc() + Duration::from_secs(60)) { + let token = self.refresh_token(&token.refresh_token).await?; + let access_token = token.access_token.to_owned(); + + { + let mut cache_token = self.inner.cache.oauth_token.write().unwrap(); + *cache_token = Some(token.clone()); + } + self.store_cache().await; + + Ok(access_token) + } else { + Ok(token.access_token.to_owned()) + } + } else { + Err(Error::Auth(AuthError::NoLogin)) + } + } } impl RustyPipeQuery { @@ -1073,6 +1355,20 @@ impl RustyPipeQuery { self } + /// Enable authentication for this request + #[must_use] + pub fn authenticated(mut self) -> Self { + self.opts.auth = Some(true); + self + } + + /// Disable authentication for this request + #[must_use] + pub fn unauthenticated(mut self) -> Self { + self.opts.auth = Some(false); + self + } + /// Set the YouTube visitor data cookie /// /// YouTube assigns a session cookie to each user which is used for personalized @@ -1123,6 +1419,15 @@ impl RustyPipeQuery { } } + /// Return `true` if the client has stored login credentials and authentication has not been disabled + pub fn auth_enabled(&self) -> bool { + if self.opts.auth == Some(false) { + return false; + } + let cache_token = self.client.inner.cache.oauth_token.read().unwrap(); + cache_token.is_some() + } + /// Create a new context object, which is included in every request to /// the YouTube API and contains language, country and device parameters. /// @@ -1473,11 +1778,15 @@ impl RustyPipeQuery { artist: ctx_src.artist, }; - let request = self + let mut r = self .request_builder(ctype, endpoint, ctx.visitor_data) - .await - .json(&req_body) - .build()?; + .await; + + if self.opts.auth == Some(true) { + let access_token = self.client.user_auth_access_token().await?; + r = r.header(header::AUTHORIZATION, format!("Bearer {}", access_token)); + } + let request = r.json(&req_body).build()?; let req_res = self.yt_request::(&request, &ctx).await?; diff --git a/src/client/player.rs b/src/client/player.rs index 4da3162..e4961aa 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -82,19 +82,22 @@ impl RustyPipeQuery { clients: &[ClientType], ) -> Result { let video_id = video_id.as_ref(); - let last_e = Error::Other("no clients".into()); + let mut last_e = Error::Other("no clients".into()); for client in clients { let res = self.player_from_client(video_id, *client).await; match res { Ok(res) => return Ok(res), Err(Error::Extraction(e)) => { - // TODO: fetch age-restricted videos with login - /* - if e.use_login() { + if e.use_login() && self.auth_enabled() { tracing::info!("{e}; fetching player with login"); - match self.player_from_client(video_id, *client).await { + match self + .clone() + .authenticated() + .player_from_client(video_id, *client) + .await + { Ok(res) => return Ok(res), Err(Error::Extraction(e)) => { if !e.switch_client() { @@ -104,8 +107,7 @@ impl RustyPipeQuery { Err(e) => return Err(e), } last_e = Error::Extraction(e); - } else*/ - if !e.switch_client() { + } else if !e.switch_client() { return Err(Error::Extraction(e)); } } diff --git a/src/error.rs b/src/error.rs index f585f28..a2125e8 100644 --- a/src/error.rs +++ b/src/error.rs @@ -16,6 +16,9 @@ pub enum Error { /// Erroneous HTTP status code received #[error("http status code: {0} message: {1}")] HttpStatus(u16, Cow<'static, str>), + /// Authentication error + #[error("auth error: {0}")] + Auth(#[from] AuthError), /// Unspecified error #[error("error: {0}")] Other(Cow<'static, str>), @@ -125,6 +128,25 @@ impl Display for UnavailabilityReason { } } +/// Error authenticating a YouTube user +#[derive(thiserror::Error, Debug)] +pub enum AuthError { + /// No user is logged in + #[error("you are not logged in")] + NoLogin, + /// The device code for user login has expired. + /// + /// Generate a new device code and try again + #[error("device code expired; try again")] + DeviceCodeExpired, + /// The access token could not be refreshed + #[error("error refreshing token: {0}; log in again")] + Refresh(String), + /// Unhandled OAuth error + #[error("unhandled OAuth error: {0}")] + Other(String), +} + pub(crate) mod internal { use super::{Error, ExtractionError}; diff --git a/tests/youtube.rs b/tests/youtube.rs index dfc1700..fe742a7 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -221,8 +221,6 @@ async fn check_video_stream(s: impl YtStream) { false, true )] -/* -TODO: add login #[case::agelimit( "ZDKQmBWTRnw", "The Rinky Pink Pounder. Hitachi Magic Wand clone teardown.", @@ -234,7 +232,6 @@ TODO: add login false, false )] -*/ #[tokio::test] #[allow(clippy::too_many_arguments)] async fn get_player( @@ -249,6 +246,11 @@ async fn get_player( #[case] is_live_content: bool, rp: RustyPipe, ) { + if id == "ZDKQmBWTRnw" && !rp.query().auth_enabled() { + tracing::warn!("unauthenticated; age-limited video cannot be tested"); + return; + } + let player_data = rp.query().player(id).await.unwrap(); let details = player_data.details; @@ -338,7 +340,7 @@ async fn get_player( #[case::members_only("vYmAhoZYg64", UnavailabilityReason::MembersOnly)] #[tokio::test] async fn get_player_error(#[case] id: &str, #[case] expect: UnavailabilityReason, rp: RustyPipe) { - let err = rp.query().player(id).await.unwrap_err(); + let err = rp.query().unauthenticated().player(id).await.unwrap_err(); match err { Error::Extraction(ExtractionError::Unavailable { reason, .. }) => { @@ -755,7 +757,7 @@ async fn get_video_details_live(rp: RustyPipe) { #[rstest] #[tokio::test] -async fn get_video_details_agegate(rp: RustyPipe) { +async fn get_video_details_agelimit(rp: RustyPipe) { let details = rp.query().video_details("ZDKQmBWTRnw").await.unwrap(); // dbg!(&details); @@ -781,7 +783,7 @@ async fn get_video_details_agegate(rp: RustyPipe) { assert!(!details.is_live); assert!(!details.is_ccommons); - // No recommendations because agegate + // No recommendations because age limit assert_eq!(details.recommended.count, Some(0)); assert!(details.recommended.items.is_empty()); }