add android/ios client, add cache

src/cache.rs

@@ -0,0 +1,177 @@
+use std::{future::Future, sync::Arc};
+
+use anyhow::Result;
+use chrono::{DateTime, Duration, Utc};
+use serde::{Deserialize, Serialize};
+use tokio::sync::Mutex;
+
+#[derive(Default, Debug)]
+pub struct Cache {
+    data: Arc<Mutex<CacheData>>,
+}
+
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
+pub struct CacheData {
+    desktop_client: Option<CacheEntry<DesktopClientData>>,
+    deobf: Option<CacheEntry<DeobfData>>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct CacheEntry<T> {
+    last_update: DateTime<Utc>,
+    data: T,
+}
+
+impl<T> From<T> for CacheEntry<T> {
+    fn from(f: T) -> Self {
+        Self {
+            last_update: Utc::now(),
+            data: f,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DesktopClientData {
+    pub client_version: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DeobfData {
+    js_url: String,
+    sig_fn: String,
+    nsig_fn: String,
+    sts: String,
+}
+
+impl Cache {
+    pub async fn get_desktop_client_data<F>(&self, updater: F) -> Result<DesktopClientData>
+    where
+        F: Future<Output = Result<DesktopClientData>> + Send + 'static,
+    {
+        let mut cache = self.data.lock().await;
+
+        if cache.desktop_client.is_none()
+            || cache.desktop_client.as_ref().unwrap().last_update < Utc::now() - Duration::days(1)
+        {
+            let cdata = updater.await?;
+            cache.desktop_client = Some(CacheEntry::from(cdata.clone()));
+            Ok(cdata)
+        } else {
+            Ok(cache.desktop_client.as_ref().unwrap().data.clone())
+        }
+    }
+
+    pub async fn get_deobf_data<F>(&self, updater: F) -> Result<DeobfData>
+    where
+        F: Future<Output = Result<DeobfData>> + Send + 'static,
+    {
+        let mut cache = self.data.lock().await;
+        if cache.deobf.is_none()
+            || cache.deobf.as_ref().unwrap().last_update < Utc::now() - Duration::days(1)
+        {
+            let deobf_data = updater.await?;
+            cache.deobf = Some(CacheEntry::from(deobf_data.clone()));
+            Ok(deobf_data)
+        } else {
+            Ok(cache.deobf.as_ref().unwrap().data.clone())
+        }
+    }
+
+    pub async fn to_json(&self) -> Result<String, serde_json::Error> {
+        let cache = self.data.lock().await;
+        serde_json::to_string(&cache.clone())
+    }
+
+    pub async fn from_json(&self, json: &str) -> Result<(), serde_json::Error> {
+        let cd = serde_json::from_str::<CacheData>(json)?;
+
+        let mut cache = self.data.lock().await;
+        *cache = cd;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn test() {
+        let cache = Cache::default();
+
+        let cdata = cache
+            .get_desktop_client_data(async {
+                Ok(DesktopClientData {
+                    client_version: "1.2.3".to_owned(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(
+            cdata,
+            DesktopClientData {
+                client_version: "1.2.3".to_owned()
+            }
+        );
+
+        let deobf_data = cache
+            .get_deobf_data(async {
+                Ok(DeobfData {
+                    js_url:
+                        "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
+                            .to_owned(),
+                    sig_fn: "t_sig_fn".to_owned(),
+                    nsig_fn: "t_nsig_fn".to_owned(),
+                    sts: "t_sts".to_owned(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(
+            deobf_data,
+            DeobfData {
+                js_url: "https://www.youtube.com/s/player/011af516/player_ias.vflset/en_US/base.js"
+                    .to_owned(),
+                sig_fn: "t_sig_fn".to_owned(),
+                nsig_fn: "t_nsig_fn".to_owned(),
+                sts: "t_sts".to_owned(),
+            }
+        );
+
+        let json = cache.to_json().await.unwrap();
+
+        let new_cache = Cache::default();
+        new_cache.from_json(&json).await.unwrap();
+
+        assert_eq!(
+            new_cache
+                .get_desktop_client_data(async {
+                    Ok(DesktopClientData {
+                        client_version: "".to_owned(),
+                    })
+                })
+                .await
+                .unwrap(),
+            cdata
+        );
+
+        assert_eq!(
+            new_cache
+                .get_deobf_data(async {
+                    Ok(DeobfData {
+                        js_url: "".to_owned(),
+                        nsig_fn: "".to_owned(),
+                        sig_fn: "".to_owned(),
+                        sts: "".to_owned(),
+                    })
+                })
+                .await
+                .unwrap(),
+            deobf_data
+        );
+    }
+}

src/client/mod.rs

@@ -6,15 +6,16 @@ use std::{sync::Arc, time::Instant};
 use anyhow::{anyhow, bail, Context, Result};
 use async_trait::async_trait;
 use fancy_regex::Regex;
-use log::{debug, error, info, warn};
+use log::{debug, warn};
 use once_cell::sync::Lazy;
 use rand::Rng;
 use reqwest::{header, Client, ClientBuilder, Method, Request, RequestBuilder, Response};
-use serde::{Serialize};
+use serde::Serialize;
 use tokio::sync::Mutex;
 
 use crate::{deobfuscate::Deobfuscator, util};
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum ClientType {
     Desktop,
     DesktopMusic,
@@ -23,15 +24,15 @@ pub enum ClientType {
     Ios,
 }
 
-#[derive(Clone, Debug, Serialize)]
+impl ClientType {
-#[serde(rename_all = "camelCase")]
+    pub fn is_web(self) -> bool {
-pub struct BaseRequest {
+        self == Self::Desktop || self == Self::DesktopMusic || self == Self::TvHtml5Embed
-    context: ContextYT,
+    }
 }
 
 #[derive(Clone, Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
-struct ContextYT {
+pub struct ContextYT {
     client: ClientInfo,
     /// only used on desktop
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -49,6 +50,8 @@ struct ClientInfo {
     client_version: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     client_screen: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    device_model: Option<String>,
     platform: String,
     #[serde(skip_serializing_if = "Option::is_none")]
     original_url: Option<String>,
@@ -95,17 +98,25 @@ const CONSENT_COOKIE: &str = "CONSENT";
 const CONSENT_COOKIE_YES: &str = "YES+yt.462272069.de+FX+";
 const CONSENT_COOKIE_NO: &str = "PENDING+";
 
-const DESKTOP_CLIENT_VERSION: &str = "2.20220721.05.00_1";
-const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
-
 const YOUTUBEI_V1_URL: &str = "https://www.youtube.com/youtubei/v1/";
 const YOUTUBEI_V1_GAPIS_URL: &str = "https://youtubei.googleapis.com/youtubei/v1/";
 
 const DISABLE_PRETTY_PRINT_PARAMETER: &str = "&prettyPrint=false";
 
+const DESKTOP_CLIENT_VERSION: &str = "2.20220721.05.00_1";
+const DESKTOP_API_KEY: &str = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+const TVHTML5_CLIENT_VERSION: &str = "2.0";
+
+const MOBILE_CLIENT_VERSION: &str = "17.10.35";
+const ANDROID_API_KEY: &str = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w";
+const IOS_API_KEY: &str = "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc";
+const IOS_DEVICE_MODEL: &str = "iPhone14,5";
+
 pub struct RustyTube {
     pub locale: Arc<Locale>,
-    pub desktop_client: DesktopClient,
+    desktop_client: Arc<DesktopClient>,
+    android_client: Arc<AndroidClient>,
+    ios_client: Arc<IosClient>,
 }
 
 #[derive(Clone)]
@@ -129,28 +140,28 @@ impl RustyTube {
 
         Self {
             locale: locale.clone(),
-            desktop_client: DesktopClient::new(locale),
+            desktop_client: Arc::new(DesktopClient::new(locale.clone())),
+            android_client: Arc::new(AndroidClient::new(locale.clone())),
+            ios_client: Arc::new(IosClient::new(locale)),
         }
     }
 
-    /*
+    pub fn get_ytclient(&self, client_type: ClientType) -> Arc<dyn YTClient> {
-    pub fn get_ytclient(&self, client_type: ClientType) -> impl YTClient {
         match client_type {
-            ClientType::Desktop => self.desktop_client,
+            ClientType::Desktop => self.desktop_client.clone(),
             ClientType::DesktopMusic => todo!(),
             ClientType::TvHtml5Embed => todo!(),
-            ClientType::Android => todo!(),
+            ClientType::Android => self.android_client.clone(),
-            ClientType::Ios => todo!(),
+            ClientType::Ios => self.ios_client.clone(),
         }
     }
-    */
 }
 
 #[async_trait]
 pub trait YTClient {
-    fn new(locale: Arc<Locale>) -> Self;
+    // fn new(locale: Arc<Locale>) -> Self;
 
-    async fn get_base_request_body(&self, localized: bool) -> BaseRequest;
+    async fn get_context(&self, localized: bool) -> ContextYT;
     async fn request_builder(&self, method: Method, url: &str) -> RequestBuilder;
     async fn exec_request(&self, request: Request) -> Result<Response>;
     async fn exec_request_text(&self, request: Request) -> Result<String>;
@@ -192,6 +203,56 @@ impl DesktopClientData {
 
 #[async_trait]
 impl YTClient for DesktopClient {
+    async fn get_context(&self, localized: bool) -> ContextYT {
+        ContextYT {
+            client: ClientInfo {
+                client_name: "WEB".to_owned(),
+                client_version: self.get_client_version().await,
+                client_screen: None,
+                device_model: None,
+                platform: "DESKTOP".to_owned(),
+                original_url: Some("https://www.youtube.com".to_owned()),
+                hl: match localized {
+                    true => self.locale.lang.to_owned(),
+                    false => "en".to_owned(),
+                },
+                gl: match localized {
+                    true => self.locale.country.to_owned(),
+                    false => "US".to_owned(),
+                },
+            },
+            request: Some(RequestYT::default()),
+            user: User::default(),
+            third_party: None,
+        }
+    }
+
+    async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder {
+        self.http
+            .request(
+                method,
+                format!(
+                    "{}{}?key={}{}",
+                    YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER
+                ),
+            )
+            .header(header::ORIGIN, "https://www.youtube.com")
+            .header(header::REFERER, "https://www.youtube.com")
+            .header(header::COOKIE, self.consent_cookie_no.to_owned())
+            .header("X-YouTube-Client-Name", "1")
+            .header("X-YouTube-Client-Version", self.get_client_version().await)
+    }
+
+    async fn exec_request(&self, request: Request) -> Result<Response> {
+        Ok(self.http.execute(request).await?.error_for_status()?)
+    }
+
+    async fn exec_request_text(&self, request: Request) -> Result<String> {
+        Ok(self.exec_request(request).await?.text().await?)
+    }
+}
+
+impl DesktopClient {
     fn new(locale: Arc<Locale>) -> Self {
         let mut rng = rand::thread_rng();
 
@@ -224,57 +285,6 @@ impl YTClient for DesktopClient {
         }
     }
 
-    async fn get_base_request_body(&self, localized: bool) -> BaseRequest {
-        BaseRequest {
-            context: ContextYT {
-                client: ClientInfo {
-                    client_name: "WEB".to_owned(),
-                    client_version: self.get_client_version().await,
-                    client_screen: None,
-                    platform: "DESKTOP".to_owned(),
-                    original_url: Some("https://www.youtube.com".to_owned()),
-                    hl: match localized {
-                        true => self.locale.lang.to_owned(),
-                        false => "en".to_owned(),
-                    },
-                    gl: match localized {
-                        true => self.locale.country.to_owned(),
-                        false => "US".to_owned(),
-                    },
-                },
-                request: Some(RequestYT::default()),
-                user: User::default(),
-                third_party: None,
-            },
-        }
-    }
-
-    async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder {
-        self.http
-            .request(
-                method,
-                format!(
-                    "{}{}?key={}{}",
-                    YOUTUBEI_V1_URL, endpoint, DESKTOP_API_KEY, DISABLE_PRETTY_PRINT_PARAMETER
-                ),
-            )
-            .header(header::ORIGIN, "https://www.youtube.com")
-            .header(header::REFERER, "https://www.youtube.com")
-            .header(header::COOKIE, self.consent_cookie_no.to_owned())
-            .header("X-YouTube-Client-Name", "1")
-            .header("X-YouTube-Client-Version", self.get_client_version().await)
-    }
-
-    async fn exec_request(&self, request: Request) -> Result<Response> {
-        Ok(self.http.execute(request).await?.error_for_status()?)
-    }
-
-    async fn exec_request_text(&self, request: Request) -> Result<String> {
-        Ok(self.exec_request(request).await?.text().await?)
-    }
-}
-
-impl DesktopClient {
     async fn extract_client_version_from_swjs(&self) -> Result<Option<String>> {
         let swjs = self
             .exec_request_text(
@@ -335,6 +345,148 @@ impl DesktopClient {
     }
 }
 
+pub struct AndroidClient {
+    locale: Arc<Locale>,
+    http: Client,
+}
+
+#[async_trait]
+impl YTClient for AndroidClient {
+    async fn get_context(&self, localized: bool) -> ContextYT {
+        ContextYT {
+            client: ClientInfo {
+                client_name: "ANDROID".to_owned(),
+                client_version: MOBILE_CLIENT_VERSION.to_owned(),
+                client_screen: None,
+                device_model: None,
+                platform: "MOBILE".to_owned(),
+                original_url: None,
+                hl: match localized {
+                    true => self.locale.lang.to_owned(),
+                    false => "en".to_owned(),
+                },
+                gl: match localized {
+                    true => self.locale.country.to_owned(),
+                    false => "US".to_owned(),
+                },
+            },
+            request: None,
+            user: User::default(),
+            third_party: None,
+        }
+    }
+
+    async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder {
+        self.http
+            .request(
+                method,
+                format!(
+                    "{}{}?key={}{}",
+                    YOUTUBEI_V1_GAPIS_URL,
+                    endpoint,
+                    ANDROID_API_KEY,
+                    DISABLE_PRETTY_PRINT_PARAMETER
+                ),
+            )
+            .header("X-Goog-Api-Format-Version", "2")
+    }
+
+    async fn exec_request(&self, request: Request) -> Result<Response> {
+        Ok(self.http.execute(request).await?.error_for_status()?)
+    }
+
+    async fn exec_request_text(&self, request: Request) -> Result<String> {
+        Ok(self.exec_request(request).await?.text().await?)
+    }
+}
+
+impl AndroidClient {
+    fn new(locale: Arc<Locale>) -> Self {
+        let http = ClientBuilder::new()
+            .user_agent(format!(
+                "com.google.android.youtube/{} (Linux; U; Android 12; {}) gzip",
+                MOBILE_CLIENT_VERSION, locale.country
+            ))
+            .gzip(true)
+            .brotli(true)
+            .build()
+            .expect("unable to build the HTTP client");
+
+        Self { locale, http }
+    }
+}
+
+pub struct IosClient {
+    locale: Arc<Locale>,
+    http: Client,
+}
+
+#[async_trait]
+impl YTClient for IosClient {
+    async fn get_context(&self, localized: bool) -> ContextYT {
+        ContextYT {
+            client: ClientInfo {
+                client_name: "IOS".to_owned(),
+                client_version: MOBILE_CLIENT_VERSION.to_owned(),
+                client_screen: None,
+                device_model: Some(IOS_DEVICE_MODEL.to_owned()),
+                platform: "MOBILE".to_owned(),
+                original_url: None,
+                hl: match localized {
+                    true => self.locale.lang.to_owned(),
+                    false => "en".to_owned(),
+                },
+                gl: match localized {
+                    true => self.locale.country.to_owned(),
+                    false => "US".to_owned(),
+                },
+            },
+            request: None,
+            user: User::default(),
+            third_party: None,
+        }
+    }
+
+    async fn request_builder(&self, method: Method, endpoint: &str) -> RequestBuilder {
+        self.http
+            .request(
+                method,
+                format!(
+                    "{}{}?key={}{}",
+                    YOUTUBEI_V1_GAPIS_URL,
+                    endpoint,
+                    ANDROID_API_KEY,
+                    DISABLE_PRETTY_PRINT_PARAMETER
+                ),
+            )
+            .header("X-Goog-Api-Format-Version", "2")
+    }
+
+    async fn exec_request(&self, request: Request) -> Result<Response> {
+        Ok(self.http.execute(request).await?.error_for_status()?)
+    }
+
+    async fn exec_request_text(&self, request: Request) -> Result<String> {
+        Ok(self.exec_request(request).await?.text().await?)
+    }
+}
+
+impl IosClient {
+    fn new(locale: Arc<Locale>) -> Self {
+        let http = ClientBuilder::new()
+            .user_agent(format!(
+                "com.google.ios.youtube/{} ({}; U; CPU iOS 15_4 like Mac OS X; {})",
+                MOBILE_CLIENT_VERSION, IOS_DEVICE_MODEL, locale.country
+            ))
+            .gzip(true)
+            .brotli(true)
+            .build()
+            .expect("unable to build the HTTP client");
+
+        Self { locale, http }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/client/player.rs

@@ -1,8 +1,8 @@
 use anyhow::{anyhow, bail, Context, Result};
 use reqwest::Method;
-use serde::{Serialize};
+use serde::Serialize;
 
-use super::{response, BaseRequest, RustyTube, YTClient};
+use super::{response, ContextYT, ClientType, RustyTube, YTClient};
 use crate::util;
 
 // REQUEST
@@ -10,13 +10,13 @@ use crate::util;
 #[derive(Clone, Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct QPlayer {
-    #[serde(flatten)]
+    context: ContextYT,
-    base: BaseRequest,
     /// Website playback context
     #[serde(skip_serializing_if = "Option::is_none")]
     playback_context: Option<QPlaybackContext>,
-    /// Content playback nonce (16 random chars)
+    /// Content playback nonce (mobile only, 16 random chars)
-    // cpn: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    cpn: Option<String>,
     /// YouTube video ID
     video_id: String,
     /// Set to true to allow extraction of streams with sensitive content
@@ -41,21 +41,37 @@ struct QContentPlaybackContext {
 }
 
 impl RustyTube {
-    pub async fn fetch_player(&self, video_id: &str) -> Result<response::Player> {
+    pub async fn fetch_player(
-        let sts = self.desktop_client.deobf.get_sts().await?;
+        &self,
+        video_id: &str,
+        client_type: ClientType,
+    ) -> Result<response::Player> {
+        let client = self.get_ytclient(client_type);
+        let context = client.get_context(false).await;
 
-        let request_body = QPlayer {
+        let request_body = if client_type.is_web() {
-            base: self.desktop_client.get_base_request_body(false).await,
+            QPlayer {
-            playback_context: Some(QPlaybackContext {
+                context,
-                content_playback_context: QContentPlaybackContext {
+                playback_context: Some(QPlaybackContext {
-                    signature_timestamp: sts,
+                    content_playback_context: QContentPlaybackContext {
-                    referer: format!("https://www.youtube.com/watch?v={}", video_id),
+                        signature_timestamp: self.desktop_client.deobf.get_sts().await?,
-                },
+                        referer: format!("https://www.youtube.com/watch?v={}", video_id),
-            }),
+                    },
-            // cpn: util::generate_content_playback_nonce(),
+                }),
-            video_id: video_id.to_owned(),
+                cpn: None,
-            content_check_ok: true,
+                video_id: video_id.to_owned(),
-            racy_check_ok: true,
+                content_check_ok: true,
+                racy_check_ok: true,
+            }
+        } else {
+            QPlayer {
+                context,
+                playback_context: None,
+                cpn: Some(util::generate_content_playback_nonce()),
+                video_id: video_id.to_owned(),
+                content_check_ok: true,
+                racy_check_ok: true,
+            }
         };
 
         let resp = self
@@ -66,6 +82,9 @@ impl RustyTube {
             .send()
             .await?;
 
+        // println!("{}", resp.text().await?);
+        // todo!();
+
         Ok(resp.json::<response::Player>().await?)
     }
 }
@@ -78,7 +97,7 @@ mod tests {
     #[test(tokio::test)]
     async fn t_fetch_stream() {
         let rt = RustyTube::new();
-        let stream = rt.fetch_player("ZeerrnuLi5E").await.unwrap();
+        let stream = rt.fetch_player("ZeerrnuLi5E", ClientType::Desktop).await.unwrap();
 
         dbg!(stream);
     }

src/client/response/player.rs

@@ -171,7 +171,6 @@ pub struct Captions {
 #[serde(rename_all = "camelCase")]
 pub struct PlayerCaptionsTracklistRenderer {
     pub caption_tracks: Vec<CaptionTrack>,
-    pub translation_languages: Vec<TranslationLanguage>,
 }
 
 #[serde_as]
@@ -184,15 +183,6 @@ pub struct CaptionTrack {
     pub language_code: String,
 }
 
-#[serde_as]
-#[derive(Clone, Debug, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct TranslationLanguage {
-    pub language_code: String,
-    #[serde_as(as = "crate::serializer::text::Text")]
-    pub language_name: String
-}
-
 #[serde_as]
 #[derive(Clone, Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]

src/lib.rs

@@ -5,6 +5,7 @@ mod macros;
 
 mod util;
 mod serializer;
+mod cache;
 mod deobfuscate;
 
 pub mod client;