Phase 1 — Foundation

Mirror NPE's dependency-free spine in Rust:

* exceptions   — NetworkError + ParsingError + ContentUnavailable
                 + ExtractionError tree, with reqwest/serde_json conversions
* localization — Localization + ContentCountry, default (en, GB)
* downloader/  — Downloader trait, Request builder, Response,
                 reqwest blocking default impl
* page         — continuation-token carrier
* image        — Image + ImageSet + ResolutionLevel
                 (HEIGHT_UNKNOWN/WIDTH_UNKNOWN = -1)
* metainfo     — title/content/url/url_text grab-bag
* service      — StreamingService trait + LinkType + ServiceInfo
* newpipe      — process-global Downloader / Localization /
                 ContentCountry singleton

Foundational invariants nailed down (per SPEC §3):
* HTTP non-2xx returns Ok(Response); only 429 throws NetworkError::Recaptcha
* Response header keys lowercase-normalized
* Request.add_header PARITY with NPE bug (silent overwrite);
  append_header is our clean addition
* default Localization is en-GB
* No cookie jar in the default downloader

Tests: 7 unit + 7 live smoke against httpbin.org (gated on
'online-tests' feature). All green.
This commit is contained in:
Sulkta 2026-05-24 16:32:36 -07:00
parent 6340b54ad3
commit 2a2367a3d0
16 changed files with 2689 additions and 1 deletions

View file

@ -0,0 +1,97 @@
// reqwest-backed default Downloader.
//
// Mirrors NewPipe-app's OkHttpDownloaderImpl behavior:
// * blocking (mirrors NPE's sync surface; async is deferred to a later
// phase that threads tokio through the whole tree)
// * no cookie jar — apps hand-build Cookie headers
// * up to 10 redirects, ~30s timeout
// * HTTP 429 → NetworkError::Recaptcha; all other status codes surface
// as Ok(Response)
use std::time::Duration;
use reqwest::blocking::Client;
use reqwest::redirect::Policy;
use crate::downloader::request::{Method, Request};
use crate::downloader::response::{Headers as RespHeaders, Response};
use crate::downloader::Downloader;
use crate::exceptions::NetworkError;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
const MAX_REDIRECTS: usize = 10;
const USER_AGENT: &str =
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36";
pub struct ReqwestDownloader {
client: Client,
}
impl ReqwestDownloader {
pub fn new() -> Result<Self, NetworkError> {
let client = Client::builder()
.user_agent(USER_AGENT)
.timeout(DEFAULT_TIMEOUT)
.redirect(Policy::limited(MAX_REDIRECTS))
.gzip(true)
.build()?;
Ok(Self { client })
}
pub fn with_client(client: Client) -> Self {
Self { client }
}
}
impl Downloader for ReqwestDownloader {
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
let method = match request.method() {
Method::Get => reqwest::Method::GET,
Method::Head => reqwest::Method::HEAD,
Method::Post => reqwest::Method::POST,
Method::Put => reqwest::Method::PUT,
Method::Delete => reqwest::Method::DELETE,
};
let mut builder = self.client.request(method, request.url());
for (name, values) in request.headers() {
for value in values {
builder = builder.header(name, value);
}
}
if let Some(loc) = request.localization() {
if request.automatic_localization_header() {
builder = builder.header("Accept-Language", loc.localization_code());
}
}
if let Some(body) = request.body() {
builder = builder.body(body.to_vec());
}
let resp = builder.send()?;
let status = resp.status();
let url_after_redirects = resp.url().to_string();
if status.as_u16() == 429 {
return Err(NetworkError::Recaptcha { url: url_after_redirects });
}
let code = status.as_u16();
let message = status.canonical_reason().unwrap_or("").to_string();
let mut headers: RespHeaders = RespHeaders::new();
for (name, value) in resp.headers().iter() {
let key = name.as_str().to_ascii_lowercase();
let val = value.to_str().unwrap_or("").to_string();
headers.entry(key).or_default().push(val);
}
let body = resp.text()?;
Ok(Response::new(code, message, headers, body, url_after_redirects))
}
}