From cddb32f190276265258c6ab45b3d43a8891c4b39 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Mon, 3 Feb 2025 02:48:54 +0100 Subject: [PATCH] feat: remove manual PO token options from downloader/cli, add new rustypipe-botguard options --- cli/README.md | 6 ++++- cli/src/main.rs | 20 +++++++++------ downloader/src/lib.rs | 57 +------------------------------------------ 3 files changed, 19 insertions(+), 64 deletions(-) diff --git a/cli/README.md b/cli/README.md index 1a6513d..e6d6c27 100644 --- a/cli/README.md +++ b/cli/README.md @@ -71,7 +71,6 @@ videos can be downloaded in parallel for improved performance. - `-c`, `--client-type` YT clients used to fetch player data (options: desktop, tv, tv-embed, android, ios; if multiple clients are specified, they are attempted in order) -- `--pot` token to circumvent bot detection ## `vdata`: Get visitor data @@ -142,6 +141,11 @@ Fetch a list of all the items saved in your YouTube/YouTube Music profile. `~/.local/share/rustypipe/rustypipe_cache.json`) - `--report-dir` Change the RustyPipe report directory location (Default: `~/.local/share/rustypipe/rustypipe_reports`) +- `--botguard-bin` Use a + [rustypipe-botguard](https://codeberg.org/ThetaDev/rustypipe-botguard) binary from the + given path for generating PO tokens +- `--no-botguard` Disable Botguard, only download videos using clients that dont require + it ### Output format diff --git a/cli/src/main.rs b/cli/src/main.rs index c3dd816..b897380 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -2,6 +2,7 @@ #![warn(clippy::todo, clippy::dbg_macro)] use std::{ + ffi::OsString, path::PathBuf, str::FromStr, sync::{atomic::AtomicUsize, Arc}, @@ -63,6 +64,12 @@ struct Cli { /// RustyPipe report folder #[clap(long, global = true)] report_dir: Option, + /// Path to rustypipe-botguard binary + #[clap(long, global = true)] + botguard_bin: Option, + /// Disable Botguard + #[clap(long, global = true)] + no_botguard: bool, } #[derive(Parser)] @@ -130,9 +137,6 @@ enum Commands { /// YT Client used to fetch player data #[clap(short, long)] client_type: Option>, - /// `pot` token to circumvent bot detection - #[clap(long)] - pot: Option, }, /// Extract video, playlist, album or channel data Get { @@ -903,6 +907,12 @@ async fn run() -> anyhow::Result<()> { if let Some(country) = cli.country { rp = rp.country(Country::from_str(&country.to_ascii_uppercase()).expect("invalid country")); } + if let Some(botguard_bin) = cli.botguard_bin { + rp = rp.botguard_bin(botguard_bin); + } + if cli.no_botguard { + rp = rp.no_botguard(); + } if cli.auth { rp = rp.authenticated(); } @@ -918,7 +928,6 @@ async fn run() -> anyhow::Result<()> { music, limit, client_type, - pot, } => { let url_target = rp.query().resolve_string(&id, false).await?; @@ -940,9 +949,6 @@ async fn run() -> anyhow::Result<()> { dl = dl.audio_tag().crop_cover(); filter = filter.no_video(); } - if let Some(pot) = pot { - dl = dl.pot(pot); - } let dl = dl.stream_filter(filter).build(); let cts = client_type.map(|c| c.into_iter().map(ClientType::from).collect::>()); diff --git a/downloader/src/lib.rs b/downloader/src/lib.rs index 29f5469..fc6cf8f 100644 --- a/downloader/src/lib.rs +++ b/downloader/src/lib.rs @@ -77,7 +77,6 @@ pub struct DownloaderBuilder { #[cfg(feature = "audiotag")] crop_cover: bool, client_types: Option>, - pot: Option, } struct DownloaderInner { @@ -109,8 +108,6 @@ struct DownloaderInner { crop_cover: bool, /// Client types for fetching videos client_types: Option>, - /// Pot token to circumvent bot detection - pot: Option, } /// Download query @@ -130,8 +127,6 @@ pub struct DownloadQuery { video_format: Option, /// Client types for fetching videos client_types: Option>, - /// Pot token to circumvent bot detection - pot: Option, } /// Video to be downloaded @@ -298,7 +293,6 @@ impl Default for DownloaderBuilder { #[cfg(feature = "audiotag")] crop_cover: false, client_types: None, - pot: None, } } } @@ -417,21 +411,6 @@ impl DownloaderBuilder { self } - /// Set the `pot` token to circumvent bot detection - /// - /// YouTube has implemented the token to prevent other clients from downloading YouTube videos. - /// The token is generated using YouTube's botguard. Therefore you need a full browser environment - /// to obtain one. - /// - /// The Invidious project has created a script to extract this token: - /// - /// The `pot` token is only used for the [`ClientType::Desktop`] and [`ClientType::DesktopMusic`] clients. - #[must_use] - pub fn pot>(mut self, pot: S) -> Self { - self.pot = Some(pot.into()); - self - } - /// Create a new, configured [`Downloader`] instance pub fn build(self) -> Downloader { self.build_with_client( @@ -466,7 +445,6 @@ impl DownloaderBuilder { #[cfg(feature = "audiotag")] crop_cover: self.crop_cover, client_types: self.client_types, - pot: self.pot, }), } } @@ -501,7 +479,6 @@ impl Downloader { filter: None, video_format: None, client_types: None, - pot: None, } } @@ -652,21 +629,6 @@ impl DownloadQuery { self } - /// Set the `pot` token to circumvent bot detection - /// - /// YouTube has implemented the token to prevent other clients from downloading YouTube videos. - /// The token is generated using YouTube's botguard. Therefore you need a full browser environment - /// to obtain one. - /// - /// The Invidious project has created a script to extract this token: - /// - /// The `pot` token is only used for the [`ClientType::Desktop`] and [`ClientType::DesktopMusic`] clients. - #[must_use] - pub fn pot>(mut self, pot: S) -> Self { - self.pot = Some(pot.into()); - self - } - /// Download the video /// /// If no download path is set, the video is downloaded to the current directory @@ -787,14 +749,6 @@ impl DownloadQuery { let player_data = q.player_from_clients(&self.video.id, &client_types).await?; let user_agent = q.user_agent(player_data.client_type); - let pot = if matches!( - player_data.client_type, - ClientType::Desktop | ClientType::DesktopMusic - ) { - self.pot.as_deref().or(self.dl.i.pot.as_deref()) - } else { - None - }; // Select streams to download let (video, audio) = player_data.select_video_audio_stream(filter); @@ -877,7 +831,6 @@ impl DownloadQuery { downloads, &self.dl.i.http, &user_agent, - pot, #[cfg(feature = "indicatif")] pb.clone(), ) @@ -1156,7 +1109,6 @@ async fn download_single_file( output: &Path, http: &Client, user_agent: &str, - pot: Option<&str>, #[cfg(feature = "indicatif")] pb: Option, ) -> Result<()> { // Check if file is already downloaded @@ -1253,7 +1205,6 @@ async fn download_single_file( size.unwrap(), offset, user_agent, - pot, #[cfg(feature = "indicatif")] pb, ) @@ -1381,7 +1332,6 @@ async fn download_chunks_by_param( size: u64, offset: u64, user_agent: &str, - pot: Option<&str>, #[cfg(feature = "indicatif")] pb: Option, ) -> Result<()> { let mut offset = offset; @@ -1394,12 +1344,9 @@ async fn download_chunks_by_param( let range = get_download_range(offset, Some(size)); tracing::debug!("Fetching range {}-{}", range.start, range.end); - let mut urlp = + let urlp = Url::parse_with_params(url, [("range", &format!("{}-{}", range.start, range.end))]) .map_err(|e| DownloadError::Progressive(format!("url parsing: {e}").into()))?; - if let Some(pot) = pot { - urlp.query_pairs_mut().append_pair("pot", pot); - } let res = http .get(urlp) @@ -1449,7 +1396,6 @@ async fn download_streams( downloads: Vec, http: &Client, user_agent: &str, - pot: Option<&str>, #[cfg(feature = "indicatif")] pb: Option, ) -> Result> { stream::iter(downloads.iter().map(Ok)) @@ -1462,7 +1408,6 @@ async fn download_streams( &d.file, http, user_agent, - pot, #[cfg(feature = "indicatif")] pb, )