// reqwest-backed default Downloader. // // Mirrors NewPipe-app's OkHttpDownloaderImpl behavior: // * blocking (mirrors NPE's sync surface; async is deferred to a later // phase that threads tokio through the whole tree) // * no cookie jar — apps hand-build Cookie headers // * up to 10 redirects, ~30s timeout // * HTTP 429 → NetworkError::Recaptcha; all other status codes surface // as Ok(Response) use std::io::Read; use std::time::Duration; use reqwest::blocking::Client; use reqwest::redirect::Policy; use crate::downloader::request::{Method, Request}; use crate::downloader::response::{Headers as RespHeaders, Response}; use crate::downloader::Downloader; use crate::exceptions::NetworkError; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); const MAX_REDIRECTS: usize = 10; const USER_AGENT: &str = "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36"; /// Hard cap on response body size. Real YT player.js is ~1.5 MB, /// InnerTube watch-page HTML is ~3 MB, channel/playlist responses /// are well under 1 MB. 32 MB leaves a generous margin while /// blocking a hostile / compromised host from blasting GB-scale /// bodies into the JVM and OOM-killing the process. const MAX_BODY_BYTES: u64 = 32 * 1024 * 1024; pub struct ReqwestDownloader { client: Client, } impl ReqwestDownloader { pub fn new() -> Result { let client = Client::builder() .user_agent(USER_AGENT) .timeout(DEFAULT_TIMEOUT) .redirect(Policy::limited(MAX_REDIRECTS)) .gzip(true) .build()?; Ok(Self { client }) } pub fn with_client(client: Client) -> Self { Self { client } } } impl Downloader for ReqwestDownloader { fn execute(&self, request: Request) -> Result { let method = match request.method() { Method::Get => reqwest::Method::GET, Method::Post => reqwest::Method::POST, }; let mut builder = self.client.request(method, request.url()); for (name, values) in request.headers() { for value in values { builder = builder.header(name, value); } } if let Some(loc) = request.localization() { if request.automatic_localization_header() { builder = builder.header("Accept-Language", loc.localization_code()); } } if let Some(body) = request.body() { builder = builder.body(body.to_vec()); } let resp = builder.send()?; let status = resp.status(); let url_after_redirects = resp.url().to_string(); if status.as_u16() == 429 { return Err(NetworkError::Recaptcha { url: url_after_redirects }); } let code = status.as_u16(); let message = status.canonical_reason().unwrap_or("").to_string(); let mut headers: RespHeaders = RespHeaders::new(); for (name, value) in resp.headers().iter() { let key = name.as_str().to_ascii_lowercase(); let val = value.to_str().unwrap_or("").to_string(); headers.entry(key).or_default().push(val); } // Fail fast when a known Content-Length already exceeds the cap. if let Some(len) = resp.content_length() { if len > MAX_BODY_BYTES { return Err(NetworkError::Transport(format!( "response body {len} bytes exceeds cap {MAX_BODY_BYTES}" ))); } } // Streaming read with hard cap. take(N+1) so we can detect // "exceeded the cap" rather than "exactly at the cap, maybe // more truncated" — if read_to_end fills MAX+1 bytes we know // the upstream had more. let mut buf: Vec = Vec::new(); let mut limited = resp.take(MAX_BODY_BYTES + 1); limited .read_to_end(&mut buf) .map_err(|e| NetworkError::Transport(format!("body read: {e}")))?; if buf.len() as u64 > MAX_BODY_BYTES { return Err(NetworkError::Transport(format!( "response body exceeded cap {MAX_BODY_BYTES}" ))); } // Reuse the body buffer on the valid-UTF-8 path (the overwhelming // common case): String::from_utf8 reinterprets the Vec in place, // whereas from_utf8_lossy always allocates + copies. Fall back to // lossy only on genuinely invalid bytes, preserving U+FFFD behavior. let body = match String::from_utf8(buf) { Ok(s) => s, Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), }; Ok(Response::new(code, message, headers, body, url_after_redirects)) } }