Phase 1 — Foundation
Mirror NPE's dependency-free spine in Rust:
* exceptions — NetworkError + ParsingError + ContentUnavailable
+ ExtractionError tree, with reqwest/serde_json conversions
* localization — Localization + ContentCountry, default (en, GB)
* downloader/ — Downloader trait, Request builder, Response,
reqwest blocking default impl
* page — continuation-token carrier
* image — Image + ImageSet + ResolutionLevel
(HEIGHT_UNKNOWN/WIDTH_UNKNOWN = -1)
* metainfo — title/content/url/url_text grab-bag
* service — StreamingService trait + LinkType + ServiceInfo
* newpipe — process-global Downloader / Localization /
ContentCountry singleton
Foundational invariants nailed down (per SPEC §3):
* HTTP non-2xx returns Ok(Response); only 429 throws NetworkError::Recaptcha
* Response header keys lowercase-normalized
* Request.add_header PARITY with NPE bug (silent overwrite);
append_header is our clean addition
* default Localization is en-GB
* No cookie jar in the default downloader
Tests: 7 unit + 7 live smoke against httpbin.org (gated on
'online-tests' feature). All green.
This commit is contained in:
parent
6340b54ad3
commit
2a2367a3d0
16 changed files with 2689 additions and 1 deletions
97
src/downloader/default_impl.rs
Normal file
97
src/downloader/default_impl.rs
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
// reqwest-backed default Downloader.
|
||||
//
|
||||
// Mirrors NewPipe-app's OkHttpDownloaderImpl behavior:
|
||||
// * blocking (mirrors NPE's sync surface; async is deferred to a later
|
||||
// phase that threads tokio through the whole tree)
|
||||
// * no cookie jar — apps hand-build Cookie headers
|
||||
// * up to 10 redirects, ~30s timeout
|
||||
// * HTTP 429 → NetworkError::Recaptcha; all other status codes surface
|
||||
// as Ok(Response)
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::redirect::Policy;
|
||||
|
||||
use crate::downloader::request::{Method, Request};
|
||||
use crate::downloader::response::{Headers as RespHeaders, Response};
|
||||
use crate::downloader::Downloader;
|
||||
use crate::exceptions::NetworkError;
|
||||
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_REDIRECTS: usize = 10;
|
||||
const USER_AGENT: &str =
|
||||
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.36";
|
||||
|
||||
pub struct ReqwestDownloader {
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl ReqwestDownloader {
|
||||
pub fn new() -> Result<Self, NetworkError> {
|
||||
let client = Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.timeout(DEFAULT_TIMEOUT)
|
||||
.redirect(Policy::limited(MAX_REDIRECTS))
|
||||
.gzip(true)
|
||||
.build()?;
|
||||
Ok(Self { client })
|
||||
}
|
||||
|
||||
pub fn with_client(client: Client) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
impl Downloader for ReqwestDownloader {
|
||||
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
||||
let method = match request.method() {
|
||||
Method::Get => reqwest::Method::GET,
|
||||
Method::Head => reqwest::Method::HEAD,
|
||||
Method::Post => reqwest::Method::POST,
|
||||
Method::Put => reqwest::Method::PUT,
|
||||
Method::Delete => reqwest::Method::DELETE,
|
||||
};
|
||||
|
||||
let mut builder = self.client.request(method, request.url());
|
||||
|
||||
for (name, values) in request.headers() {
|
||||
for value in values {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(loc) = request.localization() {
|
||||
if request.automatic_localization_header() {
|
||||
builder = builder.header("Accept-Language", loc.localization_code());
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(body) = request.body() {
|
||||
builder = builder.body(body.to_vec());
|
||||
}
|
||||
|
||||
let resp = builder.send()?;
|
||||
|
||||
let status = resp.status();
|
||||
let url_after_redirects = resp.url().to_string();
|
||||
|
||||
if status.as_u16() == 429 {
|
||||
return Err(NetworkError::Recaptcha { url: url_after_redirects });
|
||||
}
|
||||
|
||||
let code = status.as_u16();
|
||||
let message = status.canonical_reason().unwrap_or("").to_string();
|
||||
|
||||
let mut headers: RespHeaders = RespHeaders::new();
|
||||
for (name, value) in resp.headers().iter() {
|
||||
let key = name.as_str().to_ascii_lowercase();
|
||||
let val = value.to_str().unwrap_or("").to_string();
|
||||
headers.entry(key).or_default().push(val);
|
||||
}
|
||||
|
||||
let body = resp.text()?;
|
||||
|
||||
Ok(Response::new(code, message, headers, body, url_after_redirects))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue