strawcore/src/downloader/default_impl.rs

// reqwest-backed default Downloader.
//
// Mirrors NewPipe-app's OkHttpDownloaderImpl behavior:
//   * blocking (mirrors NPE's sync surface; async is deferred to a later
//     phase that threads tokio through the whole tree)
//   * no cookie jar — apps hand-build Cookie headers
//   * up to 10 redirects, ~30s timeout
//   * HTTP 429 → NetworkError::Recaptcha; all other status codes surface
//     as Ok(Response)

use std::io::Read;
use std::time::Duration;

use reqwest::blocking::Client;
use reqwest::redirect::Policy;

use crate::downloader::request::{Method, Request};
use crate::downloader::response::{Headers as RespHeaders, Response};
use crate::downloader::Downloader;
use crate::exceptions::NetworkError;

const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_REDIRECTS: usize = 10;
const USER_AGENT: &str =
    "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36";
/// Hard cap on response body size. Real YT player.js is ~1.5 MB,
/// InnerTube watch-page HTML is ~3 MB, channel/playlist responses
/// are well under 1 MB. 32 MB leaves a generous margin while
/// blocking a hostile / compromised host from blasting GB-scale
/// bodies into the JVM and OOM-killing the process.
const MAX_BODY_BYTES: u64 = 32 * 1024 * 1024;

pub struct ReqwestDownloader {
    client: Client,
}

impl ReqwestDownloader {
    pub fn new() -> Result<Self, NetworkError> {
        let client = Client::builder()
            .user_agent(USER_AGENT)
            .timeout(DEFAULT_TIMEOUT)
            .redirect(Policy::limited(MAX_REDIRECTS))
            .gzip(true)
            .build()?;
        Ok(Self { client })
    }

    pub fn with_client(client: Client) -> Self {
        Self { client }
    }
}

impl Downloader for ReqwestDownloader {
    fn execute(&self, request: Request) -> Result<Response, NetworkError> {
        let method = match request.method() {
            Method::Get => reqwest::Method::GET,
            Method::Post => reqwest::Method::POST,
        };

        let mut builder = self.client.request(method, request.url());

        for (name, values) in request.headers() {
            for value in values {
                builder = builder.header(name, value);
            }
        }

        if let Some(loc) = request.localization() {
            if request.automatic_localization_header() {
                builder = builder.header("Accept-Language", loc.localization_code());
            }
        }

        if let Some(body) = request.body() {
            builder = builder.body(body.to_vec());
        }

        let resp = builder.send()?;

        let status = resp.status();
        let url_after_redirects = resp.url().to_string();

        if status.as_u16() == 429 {
            return Err(NetworkError::Recaptcha { url: url_after_redirects });
        }

        let code = status.as_u16();
        let message = status.canonical_reason().unwrap_or("").to_string();

        let mut headers: RespHeaders = RespHeaders::new();
        for (name, value) in resp.headers().iter() {
            let key = name.as_str().to_ascii_lowercase();
            let val = value.to_str().unwrap_or("").to_string();
            headers.entry(key).or_default().push(val);
        }

        // Fail fast when a known Content-Length already exceeds the cap.
        if let Some(len) = resp.content_length() {
            if len > MAX_BODY_BYTES {
                return Err(NetworkError::Transport(format!(
                    "response body {len} bytes exceeds cap {MAX_BODY_BYTES}"
                )));
            }
        }

        // Streaming read with hard cap. take(N+1) so we can detect
        // "exceeded the cap" rather than "exactly at the cap, maybe
        // more truncated" — if read_to_end fills MAX+1 bytes we know
        // the upstream had more.
        let mut buf: Vec<u8> = Vec::new();
        let mut limited = resp.take(MAX_BODY_BYTES + 1);
        limited
            .read_to_end(&mut buf)
            .map_err(|e| NetworkError::Transport(format!("body read: {e}")))?;
        if buf.len() as u64 > MAX_BODY_BYTES {
            return Err(NetworkError::Transport(format!(
                "response body exceeded cap {MAX_BODY_BYTES}"
            )));
        }
        // Reuse the body buffer on the valid-UTF-8 path (the overwhelming
        // common case): String::from_utf8 reinterprets the Vec in place,
        // whereas from_utf8_lossy always allocates + copies. Fall back to
        // lossy only on genuinely invalid bytes, preserving U+FFFD behavior.
        let body = match String::from_utf8(buf) {
            Ok(s) => s,
            Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
        };

        Ok(Response::new(code, message, headers, body, url_after_redirects))
    }
}