diff --git a/.gitignore b/.gitignore index 3d5b6ea66..b971f9042 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,9 @@ node_modules/ !*.xcodeproj/project.xcworkspace/ !*.xcworkspace/contents.xcworkspacedata **/xcshareddata/WorkspaceSettings.xcsettings + +# Rust build artifacts +rust/target/ +strawApp/src/main/jniLibs/ +# UniFFI-generated Kotlin bindings (regen'd from .so on every build) +strawApp/src/main/java/uniffi/ diff --git a/buildSrc/src/main/kotlin/ProjectConfig.kt b/buildSrc/src/main/kotlin/ProjectConfig.kt index c8077d79a..436e62cd2 100644 --- a/buildSrc/src/main/kotlin/ProjectConfig.kt +++ b/buildSrc/src/main/kotlin/ProjectConfig.kt @@ -15,6 +15,6 @@ const val NEWPIPE_APPLICATION_ID_OLD = "org.schabi.newpipe" const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app" // Sulkta fork — Straw -const val STRAW_VERSION_CODE = 7 -const val STRAW_VERSION_NAME = "0.1.0-T" +const val STRAW_VERSION_CODE = 8 +const val STRAW_VERSION_NAME = "0.1.0-U" const val STRAW_APPLICATION_ID = "com.sulkta.straw" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 000000000..d200b06d4 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,30 @@ +# Straw Rust core — workspace. +# Hosts the JNI/UniFFI-exported library that replaces NewPipeExtractor. +# +# Builds via cargo-ndk for the four Android ABIs (arm64-v8a, armeabi-v7a, +# x86, x86_64). The Mozilla rust-android-gradle plugin wires this into the +# strawApp Gradle build so `./gradlew assembleDebug` produces a single APK +# with the .so files packed inside. + +[workspace] +resolver = "2" +members = ["strawcore"] + +[workspace.package] +edition = "2021" +license = "GPL-3.0-or-later" +authors = ["Sulkta-Coop"] +repository = "http://192.168.0.5:3001/Sulkta-Coop/straw" + +[profile.release] +# Strip debug info, run thin LTO. APK size matters more than build time here. +strip = true +lto = "thin" +codegen-units = 1 +panic = "abort" +opt-level = "z" + +[profile.dev] +# Keep debug builds fast — we're rebuilding constantly during U-1..U-5. +opt-level = 0 +debug = 1 diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 000000000..2aa515b4c --- /dev/null +++ b/rust/README.md @@ -0,0 +1,56 @@ +# `rust/` — strawcore: Rust YouTube core for Straw + +Phase **U-** of the Straw build. Goal: replace the Java NewPipeExtractor +dependency with a Rust core (rustypipe + tokio + reqwest), exposed to the +Kotlin/Compose UI via [UniFFI](https://mozilla.github.io/uniffi-rs/). +Compose UI stays in Kotlin — only the YouTube/Innertube fetching layer +moves to Rust. + +## Phases + +| Phase | What | +|---|---| +| **U-1** | Toolchain + UniFFI smoke test (`hello_from_rust`) round-tripping through JNA. No real APIs yet. | +| **U-2** | rustypipe search. `SearchViewModel` calls the Rust core. | +| **U-3** | rustypipe streamInfo + streams. `VideoDetailViewModel` + `PlayerViewModel` use it. | +| **U-4** | rustypipe channel + tabs. `ChannelViewModel` + `SubscriptionFeedViewModel`. | +| **U-5** | Rip NewPipeExtractor Java dep. Measure APK + cold-fetch latency before/after. | +| **U-6** *(stretch)* | SponsorBlock + RYD HTTP through reqwest + tokio in the same lib. | + +## Build chain + +``` +crafting-table +├── rustup stable (target add: aarch64-linux-android, armv7-linux-androideabi, +│ x86_64-linux-android, i686-linux-android) +├── cargo-ndk (cross-compile helper) +├── android-sdk (ANDROID_HOME, sdkmanager, build-tools, platforms) +└── android-ndk (ANDROID_NDK_HOME, r27c LTS at /caches/android-sdk/ndk/...) + +Gradle (strawApp/build.gradle.kts) +├── cargoBuild Exec task → cargo ndk -t ... -o jniLibs/ build --release +├── uniffiBindgen Exec task → cargo run --bin uniffi-bindgen ... --library libstrawcore.so +└── source-set wiring generated Kotlin lands in strawApp/src/main/java/uniffi/strawcore/ + +Runtime (StrawApp.onCreate) +├── System.loadLibrary("strawcore") +└── uniffi.strawcore.initLogging() +``` + +## Why UniFFI (and not raw JNI / JNA bindings) + +- Hand-written JNI: tedious, error-prone, every type change is two files + (Kotlin + Rust) that must stay in sync. +- Raw JNA: better, but you still hand-write the Kotlin side and worry about + string ownership. +- UniFFI: write Rust, annotate with `#[uniffi::export]`, get a Kotlin shim + generated. Strings, structs, enums, Result types, `async` functions all + cross the boundary transparently. The runtime is JNA under the hood. + +## When in doubt + +- `cargo check -p strawcore --target aarch64-linux-android` — fast iteration. +- `cargo run --bin uniffi-bindgen -- generate ...` — regenerate Kotlin bindings. +- `adb logcat -s strawcore` — Rust `log::info!()` lands here. +- `aapt dump badging strawApp/build/outputs/apk/debug/strawApp-debug.apk` — + inspect what ABIs/native-libs the APK carries. diff --git a/rust/strawcore/Cargo.toml b/rust/strawcore/Cargo.toml new file mode 100644 index 000000000..2b91a70c3 --- /dev/null +++ b/rust/strawcore/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "strawcore" +version = "0.1.0" +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[lib] +# cdylib — the .so that Android loads via System.loadLibrary("strawcore"). +# staticlib — kept in case we ever want to link statically for a benchmark. +crate-type = ["cdylib", "staticlib"] + +[dependencies] +# UniFFI generates the Kotlin bindings + the JNI glue. proc-macro mode +# (no .udl file) — annotate Rust fns directly with #[uniffi::export]. +# `tokio` feature wires `#[uniffi::export(async_runtime = "tokio")]` so async +# fns surface as suspend fun on the Kotlin side. +uniffi = { version = "0.28", features = ["cli", "tokio"] } +# Tokio multi-thread runtime — rustypipe is async-first. +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } +# rustypipe — the actual YouTube Innertube client. Phase U-2 wires search. +# Force rustls + webpki-roots so we don't pull openssl-sys (cross-compiling +# system OpenSSL to four Android ABIs is a tarpit; rustls is pure-Rust). +rustypipe = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots"] } +# rquickjs-sys (transitive dep of rustypipe for YT signature decryption JS) +# doesn't ship prebuilt Android bindings. Direct-depend with `bindgen` feature +# so it generates them at build time. Crafting-table has libclang preinstalled. +rquickjs-sys = { version = "0.9", default-features = false, features = ["bindgen"] } +# Error glue. +thiserror = "1" +# Single-threaded init for the runtime + extractor singletons. +once_cell = "1" +# Android log integration — `log::info!()` ends up in `adb logcat -s strawcore`. +log = "0.4" +android_logger = { version = "0.14", default-features = false } + +[build-dependencies] +uniffi = { version = "0.28", features = ["build"] } + +[[bin]] +# Generates Kotlin bindings from the compiled .so: +# cargo run --bin uniffi-bindgen generate --library target/.../libstrawcore.so \ +# --language kotlin --out-dir ../../strawApp/src/main/kotlin +name = "uniffi-bindgen" +path = "src/uniffi-bindgen.rs" diff --git a/rust/strawcore/build.rs b/rust/strawcore/build.rs new file mode 100644 index 000000000..c9431586b --- /dev/null +++ b/rust/strawcore/build.rs @@ -0,0 +1,5 @@ +// No-op build script. We're in proc-macro mode (no .udl file). The +// #[uniffi::export] / uniffi::setup_scaffolding!() macros register +// themselves at compile time. + +fn main() {} diff --git a/rust/strawcore/src/error.rs b/rust/strawcore/src/error.rs new file mode 100644 index 000000000..110b69092 --- /dev/null +++ b/rust/strawcore/src/error.rs @@ -0,0 +1,29 @@ +// Strawcore error type — anything that can fail in our Rust core surfaces +// as one of these variants. UniFFI maps it to a Kotlin sealed class +// `StrawcoreException` so Kotlin can pattern-match on the kind. + +use thiserror::Error; + +#[derive(Debug, Error, uniffi::Error)] +pub enum StrawcoreError { + #[error("network: {msg}")] + Network { msg: String }, + + #[error("extractor: {msg}")] + Extractor { msg: String }, + + #[error("not found: {what}")] + NotFound { what: String }, + + #[error("unsupported: {detail}")] + Unsupported { detail: String }, +} + +impl From for StrawcoreError { + fn from(e: rustypipe::error::Error) -> Self { + // Bucket every rustypipe error into Extractor for now. Phase U-3+ + // can pull apart specific cases (e.g. ageRestricted → NotFound, + // network timeouts → Network) when we have a tighter UI for it. + StrawcoreError::Extractor { msg: e.to_string() } + } +} diff --git a/rust/strawcore/src/lib.rs b/rust/strawcore/src/lib.rs new file mode 100644 index 000000000..848e26627 --- /dev/null +++ b/rust/strawcore/src/lib.rs @@ -0,0 +1,46 @@ +// strawcore — Rust core for the Straw Android app. +// +// Phase U-1: hello-world smoke test. One exported function, returns a +// String so we know JNI + UniFFI marshalling works end-to-end before we +// pull in rustypipe. +// +// Phase U-2+ adds real APIs (search, stream_info, channel_info). + +use std::sync::Once; + +mod error; +mod runtime; +mod search; + +#[allow(unused_imports)] +use runtime::block_on; + +// Re-exports so UniFFI sees the types at the crate root for macro discovery. +pub use error::StrawcoreError; +pub use search::SearchItem; + +/// Initialize Android logging once. The Kotlin side calls this from +/// StrawApp.onCreate() so anything emitted via `log::info!()` shows up in +/// `adb logcat -s strawcore`. +#[uniffi::export] +pub fn init_logging() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + android_logger::init_once( + android_logger::Config::default() + .with_max_level(log::LevelFilter::Info) + .with_tag("strawcore"), + ); + log::info!("strawcore initialized"); + }); +} + +/// Smoke-test entry point — round-trip a string through JNI so we know +/// the bridge is working before pulling in rustypipe. +#[uniffi::export] +pub fn hello_from_rust(name: String) -> String { + log::info!("hello_from_rust called with name={}", name); + format!("hello {} from rust 🦀 (strawcore v{})", name, env!("CARGO_PKG_VERSION")) +} + +uniffi::setup_scaffolding!(); diff --git a/rust/strawcore/src/runtime.rs b/rust/strawcore/src/runtime.rs new file mode 100644 index 000000000..52100b64c --- /dev/null +++ b/rust/strawcore/src/runtime.rs @@ -0,0 +1,21 @@ +// Tokio runtime singleton — rustypipe is async-first, so every exported +// function that touches the network needs to drive a runtime. Building one +// per call is wasteful; we build one shared multi-thread runtime at first +// use and `block_on` against it. + +use once_cell::sync::Lazy; +use tokio::runtime::Runtime; + +static RT: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) // Mobile — we don't need many. Most blocks are I/O. + .enable_all() + .thread_name("strawcore-tokio") + .build() + .expect("strawcore: failed to build tokio runtime") +}); + +#[allow(dead_code)] +pub fn block_on(fut: F) -> F::Output { + RT.block_on(fut) +} diff --git a/rust/strawcore/src/search.rs b/rust/strawcore/src/search.rs new file mode 100644 index 000000000..82d6feef4 --- /dev/null +++ b/rust/strawcore/src/search.rs @@ -0,0 +1,65 @@ +// Phase U-2 — search via rustypipe, exposed to Kotlin as a suspend fun. +// +// `SearchItem` mirrors the existing Kotlin `StreamItem` field for field so +// `SearchViewModel` can swap NewPipeExtractor for this with a one-line +// change. We map only Video items from rustypipe's result (it also returns +// channels and playlists which we don't need for the search list yet). + +use crate::error::StrawcoreError; +use rustypipe::client::RustyPipe; +use rustypipe::model::YouTubeItem; + +#[derive(Debug, Clone, uniffi::Record)] +pub struct SearchItem { + pub url: String, + pub title: String, + pub uploader: String, + pub uploader_url: Option, + pub thumbnail: Option, + /// Duration in seconds. 0 = live/unknown. + pub duration_seconds: i64, + /// Reported view count. 0 = unknown. + pub view_count: i64, +} + +fn yt_video_url(id: &str) -> String { + format!("https://www.youtube.com/watch?v={}", id) +} + +fn yt_channel_url(id: &str) -> String { + format!("https://www.youtube.com/channel/{}", id) +} + +#[uniffi::export(async_runtime = "tokio")] +pub async fn search(query: String) -> Result, StrawcoreError> { + log::info!("strawcore::search query={}", query); + let rp = RustyPipe::new(); + let results = rp.query().search(query).await?; + + let items: Vec = results + .items + .items + .into_iter() + .filter_map(|item| match item { + YouTubeItem::Video(v) => Some(SearchItem { + url: yt_video_url(&v.id), + title: v.name, + uploader: v + .channel + .as_ref() + .map(|c| c.name.clone()) + .unwrap_or_default(), + uploader_url: v.channel.as_ref().map(|c| yt_channel_url(&c.id)), + thumbnail: v.thumbnail.first().map(|t| t.url.clone()), + duration_seconds: v.duration.unwrap_or(0) as i64, + view_count: v.view_count.unwrap_or(0) as i64, + }), + // Channels + playlists are dropped at U-2; future phases can + // surface them as a "channels you might like" row. + _ => None, + }) + .collect(); + + log::info!("strawcore::search returned {} videos", items.len()); + Ok(items) +} diff --git a/rust/strawcore/src/uniffi-bindgen.rs b/rust/strawcore/src/uniffi-bindgen.rs new file mode 100644 index 000000000..e3cbdf3cf --- /dev/null +++ b/rust/strawcore/src/uniffi-bindgen.rs @@ -0,0 +1,6 @@ +// Tiny entry point so `cargo run --bin uniffi-bindgen` works — the actual +// logic lives in the uniffi crate, this just delegates to its CLI. + +fn main() { + uniffi::uniffi_bindgen_main() +} diff --git a/strawApp/build.gradle.kts b/strawApp/build.gradle.kts index 13ef39dde..0e00b8cca 100644 --- a/strawApp/build.gradle.kts +++ b/strawApp/build.gradle.kts @@ -110,4 +110,98 @@ dependencies { implementation("androidx.media3:media3-session:1.4.1") // Guava ListenableFuture support for awaiting MediaController connect. implementation("androidx.concurrent:concurrent-futures-ktx:1.2.0") + + // strawcore — Rust YouTube extractor via UniFFI/JNA. Built by the + // cargoBuild + uniffiBindgen tasks below; phase U-2+ exposes search / + // streamInfo / channelInfo to replace NewPipeExtractor. + implementation("net.java.dev.jna:jna:5.14.0@aar") } + +// ============================================================================= +// Phase U-1 — Rust core build glue. +// +// Two tasks chain into the Android build: +// cargoBuild — cross-compiles rust/strawcore for the four Android ABIs +// via cargo-ndk and drops the .so files in strawApp/src/main/jniLibs/. +// uniffiBindgen — generates the Kotlin bindings from the freshly-built lib +// into strawApp/src/main/java/uniffi/strawcore/. +// +// Both depend on: +// - cargo + rustup with the four Android targets installed +// - cargo-ndk on PATH +// - ANDROID_NDK_HOME pointing at an NDK with the right toolchains +// All of that lives in the crafting-table container (see lucy-infra +// containers/crafting-table/Dockerfile + ad-hoc install notes 2026-05-24). +// ============================================================================= + +val rustRoot = file("../rust").absolutePath +val jniLibsDir = file("src/main/jniLibs").absolutePath +val bindingsDir = file("src/main/java").absolutePath + +// Resolve cargo + the NDK by absolute path so the Gradle Exec tasks don't +// depend on whatever PATH the user invoked gradle with. Fall back to env +// var (CARGO_HOME) if set, else the crafting-table default. +val cargoHome: String = System.getenv("CARGO_HOME") ?: "/caches/cargo" +val cargoBin: String = "$cargoHome/bin/cargo" +val ndkHome: String = System.getenv("ANDROID_NDK_HOME") + ?: System.getenv("ANDROID_NDK_ROOT") + ?: "/caches/android-sdk/ndk/27.2.12479018" + +val cargoBuild by tasks.registering(Exec::class) { + group = "rust" + description = "Cross-compile strawcore for all Android ABIs via cargo-ndk." + workingDir = file(rustRoot) + environment("ANDROID_NDK_HOME", ndkHome) + environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") + commandLine = listOf( + cargoBin, "ndk", + "-t", "arm64-v8a", + "-t", "armeabi-v7a", + "-t", "x86", + "-t", "x86_64", + "-o", jniLibsDir, + "build", "--release", "-p", "strawcore", + ) + standardOutput = System.out + errorOutput = System.err +} + +// Build a host-arch debug .so for uniffi-bindgen to read metadata from. +// Cross-compiled Android .so files have the same UniFFI metadata symbols, +// but the release profile's strip+LTO can strip the sections in a way that +// trips bindgen's library-mode reader. Build host debug separately. +val cargoBuildHost by tasks.registering(Exec::class) { + group = "rust" + description = "Build host-arch debug strawcore so bindgen can read its UniFFI metadata." + workingDir = file(rustRoot) + environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") + commandLine = listOf(cargoBin, "build", "-p", "strawcore") + standardOutput = System.out + errorOutput = System.err +} + +val uniffiBindgen by tasks.registering(Exec::class) { + group = "rust" + description = "Generate Kotlin bindings for strawcore via uniffi-bindgen." + dependsOn(cargoBuildHost) + workingDir = file(rustRoot) + environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") + commandLine = listOf( + cargoBin, "run", "--quiet", "--bin", "uniffi-bindgen", "--", + "generate", + "--library", "target/debug/libstrawcore.so", + "--crate", "strawcore", + "--language", "kotlin", + "--no-format", + "--out-dir", bindingsDir, + ) + standardOutput = System.out + errorOutput = System.err +} + +// Make sure Android's JNI-libs merge picks up the freshly built .so files, +// and Kotlin compilation can resolve the generated bindings. +tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") } + .configureEach { dependsOn(cargoBuild) } +tasks.matching { it.name.startsWith("compile") && it.name.endsWith("Kotlin") } + .configureEach { dependsOn(uniffiBindgen) } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt index 6d8a1defc..f44b8027e 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt @@ -25,5 +25,14 @@ class StrawApp : Application() { History.init(this) Settings.init(this) Subscriptions.init(this) + + // Phase U-1: load the strawcore native library and route its logs + // into android logcat under the "strawcore" tag. + runCatching { + System.loadLibrary("strawcore") + uniffi.strawcore.initLogging() + }.onFailure { + android.util.Log.w("StrawApp", "strawcore not loaded: ${it.message}") + } } } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt index 5ef859be4..2a305740a 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt @@ -8,17 +8,10 @@ package com.sulkta.straw.feature.search import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.sulkta.straw.data.History -import com.sulkta.straw.util.bestThumbnail -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch -import kotlinx.coroutines.withContext -import org.schabi.newpipe.extractor.NewPipe -import org.schabi.newpipe.extractor.ServiceList -import org.schabi.newpipe.extractor.search.SearchInfo -import org.schabi.newpipe.extractor.stream.StreamInfoItem data class SearchUiState( val query: String = "", @@ -52,7 +45,23 @@ class SearchViewModel : ViewModel() { _ui.value = _ui.value.copy(loading = true, error = null, results = emptyList()) viewModelScope.launch { try { - val items = withContext(Dispatchers.IO) { search(q) } + // Phase U-2: rustypipe via UniFFI. The bindgen-generated + // `search()` is already a suspend fun running on the tokio + // runtime baked into libstrawcore.so — no Dispatchers.IO + // wrapper needed, the JNI call returns to us on the caller + // dispatcher when the future completes. + val rustItems = uniffi.strawcore.search(q) + val items = rustItems.map { r -> + StreamItem( + url = r.url, + title = r.title.ifBlank { "(no title)" }, + uploader = r.uploader, + uploaderUrl = r.uploaderUrl, + thumbnail = r.thumbnail, + durationSeconds = r.durationSeconds, + viewCount = r.viewCount, + ) + } _ui.value = _ui.value.copy(loading = false, results = items) } catch (t: Throwable) { _ui.value = _ui.value.copy( @@ -62,23 +71,4 @@ class SearchViewModel : ViewModel() { } } } - - private fun search(query: String): List { - val service = NewPipe.getService(ServiceList.YouTube.serviceId) - val qh = service.searchQHFactory.fromQuery(query, emptyList(), "") - val info = SearchInfo.getInfo(service, qh) - return info.relatedItems - .filterIsInstance() - .map { - StreamItem( - url = it.url, - title = it.name ?: "(no title)", - uploader = it.uploaderName ?: "", - uploaderUrl = it.uploaderUrl, - thumbnail = bestThumbnail(it.thumbnails), - durationSeconds = it.duration, - viewCount = it.viewCount, - ) - } - } }