feat: add OAuth user login to access age-restricted videos

.forgejo/workflows/ci.yaml

@@ -28,7 +28,9 @@ jobs:
         run: cargo clippy --all --tests --features=rss,indicatif,audiotag -- -D warnings
 
       - name: 🧪 Test
-        run: cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace
+        run: |
+          echo "${{ secrets.RUSTYPIPE_CACHE }}" > rustypipe_cache.json
+          cargo nextest run --config-file ~/.config/nextest.toml --profile ci --retries 2 --features rss --workspace
         env:
           ALL_PROXY: "http://warpproxy:8124"
 

cli/src/main.rs

@@ -191,6 +191,8 @@ enum Commands {
     },
     /// Get a YouTube visitor data cookie
     Vdata,
+    /// Log in using your Google account
+    Login,
 }
 
 #[derive(Default, Copy, Clone, ValueEnum)]
@@ -1208,6 +1210,22 @@ async fn run() -> anyhow::Result<()> {
             let vd = rp.query().get_visitor_data().await?;
             println!("{vd}");
         }
+        Commands::Login => {
+            match rp.user_auth_check_login().await {
+                Ok(_) => {}
+                Err(rustypipe::error::Error::Auth(_)) => {
+                    let device_code = rp.user_auth_get_code().await?;
+                    println!(
+                        "Open {} and enter the following code:",
+                        device_code.verification_url
+                    );
+                    anstream::println!("{}", device_code.user_code.blue());
+                    rp.user_auth_wait_for_login(&device_code).await?;
+                }
+                Err(e) => return Err(e.into()),
+            }
+            anstream::println!("{}", "Logged in.".green());
+        }
     };
     Ok(())
 }

src/client/mod.rs

@@ -24,7 +24,7 @@ mod channel_rss;
 
 use std::collections::HashMap;
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::{borrow::Cow, fmt::Debug, time::Duration};
 
 use once_cell::sync::Lazy;
@@ -32,8 +32,9 @@ use regex::Regex;
 use reqwest::{header, Client, ClientBuilder, Request, RequestBuilder, Response, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use time::OffsetDateTime;
-use tokio::sync::RwLock;
+use tokio::sync::RwLock as AsyncRwLock;
 
+use crate::error::AuthError;
 use crate::{
     cache::{CacheStorage, FileStorage, DEFAULT_CACHE_FILE},
     deobfuscate::DeobfData,
@@ -198,6 +199,87 @@ struct QContinuation<'a> {
     continuation: &'a str,
 }
 
+#[derive(Debug, Serialize)]
+struct OauthCodeRequest {
+    client_id: &'static str,
+    device_id: String,
+    device_model: &'static str,
+    scope: &'static str,
+}
+
+/// Device code used for logging a user into YouTube
+///
+/// The login process works as follows:
+/// 1. Obtain a user code and show it to the user
+/// 2. The user opens the login page under <https://google.com/device>, enters the code and logs in with his account
+/// 3. The application has to check periodically if the login has succeeded using [`RustyPipe::oauth_login`] or [`RustyPipe::oauth_wait_for_login`]
+/// 4. If the login is successful, the application receives a valid access/refresh token pair which can be used to access YouTube
+#[derive(Debug, Deserialize)]
+pub struct OauthDeviceCode {
+    device_code: String,
+    /// Code to be shown to the user to log himself in
+    pub user_code: String,
+    /// Time in seconds until the code expires
+    pub expires_in: u32,
+    /// Interval in seconds for checking if the login was completed
+    pub interval: u32,
+    /// URL to the login page (<https://google.com/device>)
+    pub verification_url: String,
+}
+
+#[derive(Debug, Serialize)]
+struct OauthTokenRequest<'a> {
+    client_id: &'static str,
+    client_secret: &'static str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    code: Option<&'a str>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    refresh_token: Option<&'a str>,
+    grant_type: &'static str,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+enum OauthTokenResponse {
+    Ok(OauthTokenResponseInner),
+    Error {
+        error: String,
+        #[serde(default)]
+        error_description: String,
+    },
+}
+
+#[derive(Debug, Deserialize)]
+struct OauthTokenResponseInner {
+    access_token: String,
+    refresh_token: Option<String>,
+    expires_in: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct OauthToken {
+    access_token: String,
+    refresh_token: String,
+    #[serde(with = "time::serde::rfc3339")]
+    expires_at: OffsetDateTime,
+}
+
+impl OauthToken {
+    fn from_response(
+        value: OauthTokenResponseInner,
+        refresh_token: Option<String>,
+    ) -> Result<Self, Error> {
+        Ok(Self {
+            access_token: value.access_token,
+            refresh_token: value
+                .refresh_token
+                .or(refresh_token)
+                .ok_or(Error::Other("missing refresh token".into()))?,
+            expires_at: util::now_sec() + Duration::from_secs(value.expires_in.into()),
+        })
+    }
+}
+
 const DEFAULT_UA: &str = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0";
 const MOBILE_UA: &str = "Mozilla/5.0 (Android 14; Mobile; rv:129.0) Gecko/129.0 Firefox/129.0";
 const TV_UA: &str = "Mozilla/5.0 (SMART-TV; Linux; Tizen 5.0) AppleWebKit/538.1 (KHTML, like Gecko) Version/5.0 NativeTVAds Safari/538.1";
@@ -225,6 +307,11 @@ const TV_CLIENT_VERSION: &str = "7.20241008.14.02";
 const APP_CLIENT_VERSION: &str = "18.03.33";
 const IOS_DEVICE_MODEL: &str = "iPhone14,5";
 
+const OAUTH_CLIENT_ID: &str =
+    "861556708454-d6dlm3lh05idd8npek18k6be8ba3oc68.apps.googleusercontent.com";
+const OAUTH_CLIENT_SECRET: &str = "SboVhoG9s0rNafixCSGGKXAT";
+const OAUTH_SCOPES: &str = "http://gdata.youtube.com https://www.googleapis.com/auth/youtube";
+
 static CLIENT_VERSION_REGEX: Lazy<Regex> =
     Lazy::new(|| Regex::new(r#""INNERTUBE_CONTEXT_CLIENT_VERSION":"([\w\d\._-]+?)""#).unwrap());
 static VISITOR_DATA_REGEX: Lazy<Regex> =
@@ -263,6 +350,7 @@ struct RustyPipeOpts {
     country: Country,
     report: bool,
     strict: bool,
+    auth: Option<bool>,
     visitor_data: Option<String>,
 }
 
@@ -377,6 +465,7 @@ impl Default for RustyPipeOpts {
             country: Country::Us,
             report: false,
             strict: false,
+            auth: None,
             visitor_data: None,
         }
     }
@@ -384,8 +473,9 @@ impl Default for RustyPipeOpts {
 
 #[derive(Default, Debug)]
 struct CacheHolder {
-    clients: HashMap<ClientType, RwLock<CacheEntry<ClientData>>>,
+    clients: HashMap<ClientType, AsyncRwLock<CacheEntry<ClientData>>>,
-    deobf: RwLock<CacheEntry<DeobfData>>,
+    deobf: AsyncRwLock<CacheEntry<DeobfData>>,
+    oauth_token: RwLock<Option<OauthToken>>,
 }
 
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
@@ -394,6 +484,8 @@ struct CacheData {
     clients: HashMap<ClientType, CacheEntry<ClientData>>,
     #[serde(skip_serializing_if = "CacheEntry::is_none")]
     deobf: CacheEntry<DeobfData>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    oauth_token: Option<OauthToken>,
 }
 
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
@@ -532,7 +624,12 @@ impl RustyPipeBuilder {
             ClientType::Tv,
         ]
         .into_iter()
-        .map(|c| (c, RwLock::new(cdata.clients.remove(&c).unwrap_or_default())))
+        .map(|c| {
+            (
+                c,
+                AsyncRwLock::new(cdata.clients.remove(&c).unwrap_or_default()),
+            )
+        })
         .collect::<HashMap<_, _>>();
 
         Ok(RustyPipe {
@@ -547,7 +644,8 @@ impl RustyPipeBuilder {
                 n_http_retries: self.n_http_retries,
                 cache: CacheHolder {
                     clients: cache_clients,
-                    deobf: RwLock::new(cdata.deobf),
+                    deobf: AsyncRwLock::new(cdata.deobf),
+                    oauth_token: RwLock::new(cdata.oauth_token),
                 },
                 default_opts: self.default_opts,
                 user_agent,
@@ -693,6 +791,20 @@ impl RustyPipeBuilder {
         self
     }
 
+    /// Enable authentication for all requests
+    #[must_use]
+    pub fn authenticated(mut self) -> Self {
+        self.default_opts.auth = Some(true);
+        self
+    }
+
+    /// Disable authentication for all requests
+    #[must_use]
+    pub fn unauthenticated(mut self) -> Self {
+        self.default_opts.auth = Some(false);
+        self
+    }
+
     /// Set the YouTube visitor data cookie
     ///
     /// YouTube assigns a session cookie to each user which is used for personalized
@@ -970,6 +1082,7 @@ impl RustyPipe {
             let cdata = CacheData {
                 clients: cache_clients,
                 deobf: self.inner.cache.deobf.read().await.clone(),
+                oauth_token: self.inner.cache.oauth_token.read().unwrap().clone(),
             };
 
             match serde_json::to_string(&cdata) {
@@ -1033,6 +1146,175 @@ impl RustyPipe {
             }
         }
     }
+
+    /// Get a new device code for logging into YouTube
+    pub async fn user_auth_get_code(&self) -> Result<OauthDeviceCode, Error> {
+        tracing::debug!("getting OAuth user code");
+
+        let code_request = OauthCodeRequest {
+            client_id: OAUTH_CLIENT_ID,
+            device_id: util::random_uuid(),
+            device_model: "ytlr:samsung:smarttv",
+            scope: OAUTH_SCOPES,
+        };
+
+        self.inner
+            .http
+            .post("https://www.youtube.com/o/oauth2/device/code")
+            .header(header::USER_AGENT, TV_UA)
+            .header(header::ORIGIN, YOUTUBE_HOME_URL)
+            .header(header::REFERER, YOUTUBE_TV_URL)
+            .json(&code_request)
+            .send()
+            .await?
+            .error_for_status()?
+            .json::<OauthDeviceCode>()
+            .await
+            .map_err(Error::from)
+    }
+
+    /// Attempt to log in the user using the given device code
+    ///
+    /// Returns `true` if the user has successfully logged in using the code.
+    ///
+    /// Returns `false` if the user has not logged in yet, in this case repeat
+    /// the login attempt after a few seconds.
+    /// The function [`RustyPipe::oauth_wait_for_login`] does this automatically.
+    pub async fn user_auth_login(&self, code: &OauthDeviceCode) -> Result<bool, Error> {
+        tracing::debug!("OAuth login attempt (user_code: {})", code.user_code);
+
+        let token_request = OauthTokenRequest {
+            client_id: OAUTH_CLIENT_ID,
+            client_secret: OAUTH_CLIENT_SECRET,
+            code: Some(&code.device_code),
+            refresh_token: None,
+            grant_type: "http://oauth.net/grant_type/device/1.0",
+        };
+
+        let token_response = self
+            .inner
+            .http
+            .post("https://www.youtube.com/o/oauth2/token")
+            .header(header::USER_AGENT, TV_UA)
+            .header(header::ORIGIN, YOUTUBE_HOME_URL)
+            .header(header::REFERER, YOUTUBE_TV_URL)
+            .json(&token_request)
+            .send()
+            .await?
+            .error_for_status()?
+            .json::<OauthTokenResponse>()
+            .await?;
+
+        match token_response {
+            OauthTokenResponse::Ok(token) => {
+                let token = OauthToken::from_response(token, None)?;
+                {
+                    let mut cache_token = self.inner.cache.oauth_token.write().unwrap();
+                    *cache_token = Some(token);
+                }
+                self.store_cache().await;
+                Ok(true)
+            }
+            OauthTokenResponse::Error {
+                error,
+                error_description,
+            } => match error.as_str() {
+                "authorization_pending" => Ok(false),
+                "expired_token" => Err(Error::Auth(AuthError::DeviceCodeExpired)),
+                _ => Err(Error::Auth(AuthError::Other(format!(
+                    "{error}: {error_description}"
+                )))),
+            },
+        }
+    }
+
+    /// Attempt to refresh the OAuth access token to check if the user is successfully logged in
+    /// and the session is still valid.
+    pub async fn user_auth_check_login(&self) -> Result<(), Error> {
+        let cache_token = self.inner.cache.oauth_token.read().unwrap().clone();
+        if let Some(token) = cache_token {
+            let token = self.refresh_token(&token.refresh_token).await?;
+            {
+                let mut cache_token = self.inner.cache.oauth_token.write().unwrap();
+                *cache_token = Some(token.clone());
+            }
+            self.store_cache().await;
+            Ok(())
+        } else {
+            Err(Error::Auth(AuthError::NoLogin))
+        }
+    }
+
+    /// Attempt to log in the user using the given device code.
+    ///
+    /// This function waits until the login was successful or an error occurred.
+    pub async fn user_auth_wait_for_login(&self, code: &OauthDeviceCode) -> Result<(), Error> {
+        while !self.user_auth_login(code).await? {
+            tokio::time::sleep(Duration::from_secs(code.interval.into())).await;
+        }
+        Ok(())
+    }
+
+    async fn refresh_token(&self, refresh_token: &str) -> Result<OauthToken, Error> {
+        tracing::debug!("refreshing OAuth token");
+
+        let token_request = OauthTokenRequest {
+            client_id: OAUTH_CLIENT_ID,
+            client_secret: OAUTH_CLIENT_SECRET,
+            code: None,
+            refresh_token: Some(refresh_token),
+            grant_type: "refresh_token",
+        };
+
+        let token_response = self
+            .inner
+            .http
+            .post("https://www.youtube.com/o/oauth2/token")
+            .header(header::USER_AGENT, TV_UA)
+            .header(header::ORIGIN, YOUTUBE_HOME_URL)
+            .header(header::REFERER, YOUTUBE_TV_URL)
+            .json(&token_request)
+            .send()
+            .await?
+            .error_for_status()?
+            .json::<OauthTokenResponse>()
+            .await?;
+
+        match token_response {
+            OauthTokenResponse::Ok(token) => {
+                OauthToken::from_response(token, Some(refresh_token.to_owned()))
+            }
+            OauthTokenResponse::Error {
+                error,
+                error_description,
+            } => Err(Error::Auth(AuthError::Refresh(format!(
+                "{error}: {error_description}"
+            )))),
+        }
+    }
+
+    /// Get the OAuth access token for accessing YouTube as an authenticated user
+    pub async fn user_auth_access_token(&self) -> Result<String, Error> {
+        let cache_token = self.inner.cache.oauth_token.read().unwrap().clone();
+        if let Some(token) = cache_token {
+            if token.expires_at < (OffsetDateTime::now_utc() + Duration::from_secs(60)) {
+                let token = self.refresh_token(&token.refresh_token).await?;
+                let access_token = token.access_token.to_owned();
+
+                {
+                    let mut cache_token = self.inner.cache.oauth_token.write().unwrap();
+                    *cache_token = Some(token.clone());
+                }
+                self.store_cache().await;
+
+                Ok(access_token)
+            } else {
+                Ok(token.access_token.to_owned())
+            }
+        } else {
+            Err(Error::Auth(AuthError::NoLogin))
+        }
+    }
 }
 
 impl RustyPipeQuery {
@@ -1073,6 +1355,20 @@ impl RustyPipeQuery {
         self
     }
 
+    /// Enable authentication for this request
+    #[must_use]
+    pub fn authenticated(mut self) -> Self {
+        self.opts.auth = Some(true);
+        self
+    }
+
+    /// Disable authentication for this request
+    #[must_use]
+    pub fn unauthenticated(mut self) -> Self {
+        self.opts.auth = Some(false);
+        self
+    }
+
     /// Set the YouTube visitor data cookie
     ///
     /// YouTube assigns a session cookie to each user which is used for personalized
@@ -1123,6 +1419,15 @@ impl RustyPipeQuery {
         }
     }
 
+    /// Return `true` if the client has stored login credentials and authentication has not been disabled
+    pub fn auth_enabled(&self) -> bool {
+        if self.opts.auth == Some(false) {
+            return false;
+        }
+        let cache_token = self.client.inner.cache.oauth_token.read().unwrap();
+        cache_token.is_some()
+    }
+
     /// Create a new context object, which is included in every request to
     /// the YouTube API and contains language, country and device parameters.
     ///
@@ -1473,11 +1778,15 @@ impl RustyPipeQuery {
             artist: ctx_src.artist,
         };
 
-        let request = self
+        let mut r = self
             .request_builder(ctype, endpoint, ctx.visitor_data)
-            .await
+            .await;
-            .json(&req_body)
+
-            .build()?;
+        if self.opts.auth == Some(true) {
+            let access_token = self.client.user_auth_access_token().await?;
+            r = r.header(header::AUTHORIZATION, format!("Bearer {}", access_token));
+        }
+        let request = r.json(&req_body).build()?;
 
         let req_res = self.yt_request::<R, M>(&request, &ctx).await?;
 

src/client/player.rs

@@ -82,19 +82,22 @@ impl RustyPipeQuery {
         clients: &[ClientType],
     ) -> Result<VideoPlayer, Error> {
         let video_id = video_id.as_ref();
-        let last_e = Error::Other("no clients".into());
+        let mut last_e = Error::Other("no clients".into());
 
         for client in clients {
             let res = self.player_from_client(video_id, *client).await;
             match res {
                 Ok(res) => return Ok(res),
                 Err(Error::Extraction(e)) => {
-                    // TODO: fetch age-restricted videos with login
+                    if e.use_login() && self.auth_enabled() {
-                    /*
-                    if e.use_login() {
                         tracing::info!("{e}; fetching player with login");
 
-                        match self.player_from_client(video_id, *client).await {
+                        match self
+                            .clone()
+                            .authenticated()
+                            .player_from_client(video_id, *client)
+                            .await
+                        {
                             Ok(res) => return Ok(res),
                             Err(Error::Extraction(e)) => {
                                 if !e.switch_client() {
@@ -104,8 +107,7 @@ impl RustyPipeQuery {
                             Err(e) => return Err(e),
                         }
                         last_e = Error::Extraction(e);
-                    } else*/
+                    } else if !e.switch_client() {
-                    if !e.switch_client() {
                         return Err(Error::Extraction(e));
                     }
                 }

src/error.rs

@@ -16,6 +16,9 @@ pub enum Error {
     /// Erroneous HTTP status code received
     #[error("http status code: {0} message: {1}")]
     HttpStatus(u16, Cow<'static, str>),
+    /// Authentication error
+    #[error("auth error: {0}")]
+    Auth(#[from] AuthError),
     /// Unspecified error
     #[error("error: {0}")]
     Other(Cow<'static, str>),
@@ -125,6 +128,25 @@ impl Display for UnavailabilityReason {
     }
 }
 
+/// Error authenticating a YouTube user
+#[derive(thiserror::Error, Debug)]
+pub enum AuthError {
+    /// No user is logged in
+    #[error("you are not logged in")]
+    NoLogin,
+    /// The device code for user login has expired.
+    ///
+    /// Generate a new device code and try again
+    #[error("device code expired; try again")]
+    DeviceCodeExpired,
+    /// The access token could not be refreshed
+    #[error("error refreshing token: {0}; log in again")]
+    Refresh(String),
+    /// Unhandled OAuth error
+    #[error("unhandled OAuth error: {0}")]
+    Other(String),
+}
+
 pub(crate) mod internal {
     use super::{Error, ExtractionError};
 

tests/youtube.rs

@@ -221,8 +221,6 @@ async fn check_video_stream(s: impl YtStream) {
     false,
     true
 )]
-/*
-TODO: add login
 #[case::agelimit(
     "ZDKQmBWTRnw",
     "The Rinky Pink Pounder.  Hitachi Magic Wand clone teardown.",
@@ -234,7 +232,6 @@ TODO: add login
     false,
     false
 )]
-*/
 #[tokio::test]
 #[allow(clippy::too_many_arguments)]
 async fn get_player(
@@ -249,6 +246,11 @@ async fn get_player(
     #[case] is_live_content: bool,
     rp: RustyPipe,
 ) {
+    if id == "ZDKQmBWTRnw" && !rp.query().auth_enabled() {
+        tracing::warn!("unauthenticated; age-limited video cannot be tested");
+        return;
+    }
+
     let player_data = rp.query().player(id).await.unwrap();
     let details = player_data.details;
 
@@ -338,7 +340,7 @@ async fn get_player(
 #[case::members_only("vYmAhoZYg64", UnavailabilityReason::MembersOnly)]
 #[tokio::test]
 async fn get_player_error(#[case] id: &str, #[case] expect: UnavailabilityReason, rp: RustyPipe) {
-    let err = rp.query().player(id).await.unwrap_err();
+    let err = rp.query().unauthenticated().player(id).await.unwrap_err();
 
     match err {
         Error::Extraction(ExtractionError::Unavailable { reason, .. }) => {
@@ -755,7 +757,7 @@ async fn get_video_details_live(rp: RustyPipe) {
 
 #[rstest]
 #[tokio::test]
-async fn get_video_details_agegate(rp: RustyPipe) {
+async fn get_video_details_agelimit(rp: RustyPipe) {
     let details = rp.query().video_details("ZDKQmBWTRnw").await.unwrap();
 
     // dbg!(&details);
@@ -781,7 +783,7 @@ async fn get_video_details_agegate(rp: RustyPipe) {
     assert!(!details.is_live);
     assert!(!details.is_ccommons);
 
-    // No recommendations because agegate
+    // No recommendations because age limit
     assert_eq!(details.recommended.count, Some(0));
     assert!(details.recommended.items.is_empty());
 }