diff --git a/buildSrc/src/main/kotlin/ProjectConfig.kt b/buildSrc/src/main/kotlin/ProjectConfig.kt index 5f3a39428..6f6f42f73 100644 --- a/buildSrc/src/main/kotlin/ProjectConfig.kt +++ b/buildSrc/src/main/kotlin/ProjectConfig.kt @@ -15,6 +15,6 @@ const val NEWPIPE_APPLICATION_ID_OLD = "org.schabi.newpipe" const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app" // Sulkta fork — Straw -const val STRAW_VERSION_CODE = 11 -const val STRAW_VERSION_NAME = "0.1.0-W2" +const val STRAW_VERSION_CODE = 12 +const val STRAW_VERSION_NAME = "0.1.0-X" const val STRAW_APPLICATION_ID = "com.sulkta.straw" diff --git a/rust/strawcore/src/stream.rs b/rust/strawcore/src/stream.rs index ea35ac76b..8f550b7f3 100644 --- a/rust/strawcore/src/stream.rs +++ b/rust/strawcore/src/stream.rs @@ -129,14 +129,18 @@ pub async fn stream_info(url: String) -> Result { log::info!("strawcore::stream_info id={}", id); let rp = RustyPipe::new(); - // rustypipe's default `player()` uses the Web client first. Those URLs - // come back signed against the Web fetch's session/UA — ExoPlayer can't - // replay them (404/403/black screen). Force the TV embedded + iOS - // clients, both of which return ungated direct-play URLs the way - // NewPipe's resolver does. + // rustypipe's default `player()` uses the Web client first, which + // returns signed URLs that need JS deobfuscation. Even the TV (TVHTML5) + // client signs URLs nowadays, so deobfuscation runs and currently + // fails ("could not extract sig fn name") because YT changed the + // obfuscation pattern after rustypipe 0.11.4's last cut. + // + // Android and iOS YT-app clients serve URLs UNSIGNED — no sig + // decryption needed, ExoPlayer plays them directly. This is the same + // path NewPipe uses for its mobile + iOS-embed strategies. let player = rp .query() - .player_from_clients(&id, &[ClientType::Tv, ClientType::Ios]) + .player_from_clients(&id, &[ClientType::Android, ClientType::Ios]) .await?; let details = &player.details; diff --git a/strawApp/build.gradle.kts b/strawApp/build.gradle.kts index c8660fc85..13ef39dde 100644 --- a/strawApp/build.gradle.kts +++ b/strawApp/build.gradle.kts @@ -94,8 +94,8 @@ dependencies { implementation(libs.coil.compose) implementation(libs.coil.network.okhttp) - // Phase U-5: NewPipeExtractor (Java) torn out — strawcore handles - // all YT extraction. OkHttp stays for RYD + SponsorBlock JSON clients. + // NewPipeExtractor (JVM/Android-only) + its OkHttp dep + implementation(libs.newpipe.extractor) implementation(libs.squareup.okhttp) // JSON for SponsorBlock + Return YouTube Dislike clients @@ -110,98 +110,4 @@ dependencies { implementation("androidx.media3:media3-session:1.4.1") // Guava ListenableFuture support for awaiting MediaController connect. implementation("androidx.concurrent:concurrent-futures-ktx:1.2.0") - - // strawcore — Rust YouTube extractor via UniFFI/JNA. Built by the - // cargoBuild + uniffiBindgen tasks below; phase U-2+ exposes search / - // streamInfo / channelInfo to replace NewPipeExtractor. - implementation("net.java.dev.jna:jna:5.14.0@aar") } - -// ============================================================================= -// Phase U-1 — Rust core build glue. -// -// Two tasks chain into the Android build: -// cargoBuild — cross-compiles rust/strawcore for the four Android ABIs -// via cargo-ndk and drops the .so files in strawApp/src/main/jniLibs/. -// uniffiBindgen — generates the Kotlin bindings from the freshly-built lib -// into strawApp/src/main/java/uniffi/strawcore/. -// -// Both depend on: -// - cargo + rustup with the four Android targets installed -// - cargo-ndk on PATH -// - ANDROID_NDK_HOME pointing at an NDK with the right toolchains -// All of that lives in the crafting-table container (see lucy-infra -// containers/crafting-table/Dockerfile + ad-hoc install notes 2026-05-24). -// ============================================================================= - -val rustRoot = file("../rust").absolutePath -val jniLibsDir = file("src/main/jniLibs").absolutePath -val bindingsDir = file("src/main/java").absolutePath - -// Resolve cargo + the NDK by absolute path so the Gradle Exec tasks don't -// depend on whatever PATH the user invoked gradle with. Fall back to env -// var (CARGO_HOME) if set, else the crafting-table default. -val cargoHome: String = System.getenv("CARGO_HOME") ?: "/caches/cargo" -val cargoBin: String = "$cargoHome/bin/cargo" -val ndkHome: String = System.getenv("ANDROID_NDK_HOME") - ?: System.getenv("ANDROID_NDK_ROOT") - ?: "/caches/android-sdk/ndk/27.2.12479018" - -val cargoBuild by tasks.registering(Exec::class) { - group = "rust" - description = "Cross-compile strawcore for all Android ABIs via cargo-ndk." - workingDir = file(rustRoot) - environment("ANDROID_NDK_HOME", ndkHome) - environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") - commandLine = listOf( - cargoBin, "ndk", - "-t", "arm64-v8a", - "-t", "armeabi-v7a", - "-t", "x86", - "-t", "x86_64", - "-o", jniLibsDir, - "build", "--release", "-p", "strawcore", - ) - standardOutput = System.out - errorOutput = System.err -} - -// Build a host-arch debug .so for uniffi-bindgen to read metadata from. -// Cross-compiled Android .so files have the same UniFFI metadata symbols, -// but the release profile's strip+LTO can strip the sections in a way that -// trips bindgen's library-mode reader. Build host debug separately. -val cargoBuildHost by tasks.registering(Exec::class) { - group = "rust" - description = "Build host-arch debug strawcore so bindgen can read its UniFFI metadata." - workingDir = file(rustRoot) - environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") - commandLine = listOf(cargoBin, "build", "-p", "strawcore") - standardOutput = System.out - errorOutput = System.err -} - -val uniffiBindgen by tasks.registering(Exec::class) { - group = "rust" - description = "Generate Kotlin bindings for strawcore via uniffi-bindgen." - dependsOn(cargoBuildHost) - workingDir = file(rustRoot) - environment("PATH", "$cargoHome/bin:${System.getenv("PATH") ?: ""}") - commandLine = listOf( - cargoBin, "run", "--quiet", "--bin", "uniffi-bindgen", "--", - "generate", - "--library", "target/debug/libstrawcore.so", - "--crate", "strawcore", - "--language", "kotlin", - "--no-format", - "--out-dir", bindingsDir, - ) - standardOutput = System.out - errorOutput = System.err -} - -// Make sure Android's JNI-libs merge picks up the freshly built .so files, -// and Kotlin compilation can resolve the generated bindings. -tasks.matching { it.name.startsWith("merge") && it.name.endsWith("JniLibFolders") } - .configureEach { dependsOn(cargoBuild) } -tasks.matching { it.name.startsWith("compile") && it.name.endsWith("Kotlin") } - .configureEach { dependsOn(uniffiBindgen) } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt index 3eeabcce7..6d8a1defc 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt @@ -1,9 +1,6 @@ /* * SPDX-FileCopyrightText: 2026 Sulkta-Coop * SPDX-License-Identifier: GPL-3.0-or-later - * - * Phase U-5: NewPipeExtractor (Java) torn out. All YT extraction goes - * through strawcore (Rust + rustypipe + UniFFI) now. */ package com.sulkta.straw @@ -12,21 +9,21 @@ import android.app.Application import com.sulkta.straw.data.History import com.sulkta.straw.data.Settings import com.sulkta.straw.data.Subscriptions +import com.sulkta.straw.extractor.NewPipeDownloader +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.localization.ContentCountry +import org.schabi.newpipe.extractor.localization.Localization class StrawApp : Application() { override fun onCreate() { super.onCreate() + NewPipe.init( + NewPipeDownloader.init(), + Localization("en", "US"), + ContentCountry("US"), + ) History.init(this) Settings.init(this) Subscriptions.init(this) - - // Load strawcore native + route its logs into android logcat under - // the "strawcore" tag. - runCatching { - System.loadLibrary("strawcore") - uniffi.strawcore.initLogging() - }.onFailure { - android.util.Log.w("StrawApp", "strawcore not loaded: ${it.message}") - } } } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt b/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt new file mode 100644 index 000000000..bc185b5e0 --- /dev/null +++ b/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2026 Sulkta-Coop + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Minimal OkHttp-backed implementation of NewPipeExtractor's Downloader. + * No cookies, no recaptcha handling — anonymous browsing only. Modeled after + * NewPipe's DownloaderImpl but trimmed down for fork scope. + */ + +package com.sulkta.straw.extractor + +import com.sulkta.straw.net.NEWPIPE_MAX_BYTES +import com.sulkta.straw.net.cappedString +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import org.schabi.newpipe.extractor.downloader.Downloader +import org.schabi.newpipe.extractor.downloader.Request +import org.schabi.newpipe.extractor.downloader.Response +import java.io.IOException +import java.util.concurrent.TimeUnit + +class NewPipeDownloader private constructor( + private val client: OkHttpClient, +) : Downloader() { + + override fun execute(request: Request): Response { + val httpMethod = request.httpMethod() + val url = request.url() + val headers = request.headers() + val data: ByteArray? = request.dataToSend() + + val requestBody = data?.toRequestBody(null) + + val okBuilder = okhttp3.Request.Builder() + .method(httpMethod, requestBody) + .url(url) + + // AUD-HIGH: copy NPE headers BEFORE adding our explicit UA so the + // explicit UA wins; guard against header values containing \r/\n + // which OkHttp's addHeader rejects via IAE (turning a poisoned + // response into an app crash). + headers.forEach { (name, values) -> + if (name.equals("User-Agent", ignoreCase = true)) return@forEach + okBuilder.removeHeader(name) + values.forEach { value -> + runCatching { okBuilder.addHeader(name, value) } + } + } + okBuilder.removeHeader("User-Agent") + okBuilder.addHeader("User-Agent", USER_AGENT) + + val okResponse = client.newCall(okBuilder.build()).execute() + val body = okResponse.body + // AUD-HIGH: bounded read to defend against OOM via gigabyte response. + val bodyString = body?.cappedString(NEWPIPE_MAX_BYTES) ?: "" + val responseHeaders = okResponse.headers.toMultimap() + val latestUrl = okResponse.request.url.toString() + if (okResponse.code == 429) { + okResponse.close() + throw IOException("HTTP 429 — rate limited") + } + okResponse.close() + + return Response( + okResponse.code, + okResponse.message, + responseHeaders, + bodyString, + latestUrl, + ) + } + + companion object { + const val USER_AGENT = + "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/120.0.0.0 Mobile Safari/537.36" + + @Volatile private var instance: NewPipeDownloader? = null + + fun init(builder: OkHttpClient.Builder? = null): NewPipeDownloader { + val client = (builder ?: OkHttpClient.Builder()) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build() + val d = NewPipeDownloader(client) + instance = d + return d + } + + fun get(): NewPipeDownloader = instance + ?: error("NewPipeDownloader not initialized — call init() first") + + fun client(): OkHttpClient = get().client + } +} diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/channel/ChannelViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/channel/ChannelViewModel.kt index fcc1d81b1..e3cb6b2e4 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/channel/ChannelViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/channel/ChannelViewModel.kt @@ -1,10 +1,6 @@ /* * SPDX-FileCopyrightText: 2026 Sulkta-Coop * SPDX-License-Identifier: GPL-3.0-or-later - * - * Phase U-4: ChannelInfo + Videos tab moved to strawcore (rustypipe). - * The two separate ChannelInfo.getInfo + ChannelTabInfo.getInfo calls - * collapse into one Rust round-trip. */ package com.sulkta.straw.feature.channel @@ -12,10 +8,19 @@ package com.sulkta.straw.feature.channel import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.sulkta.straw.feature.search.StreamItem +import com.sulkta.straw.util.bestThumbnail +import kotlinx.coroutines.Dispatchers +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.channel.ChannelInfo +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs +import org.schabi.newpipe.extractor.stream.StreamInfoItem data class ChannelUiState( val loading: Boolean = true, @@ -35,24 +40,43 @@ class ChannelViewModel : ViewModel() { _ui.value = ChannelUiState(loading = true) viewModelScope.launch { try { - val ch = uniffi.strawcore.channelInfo(channelUrl) - val videos = ch.videos.map { v -> - StreamItem( - url = v.url, - title = v.title.ifBlank { "(no title)" }, - uploader = v.uploader, - uploaderUrl = v.uploaderUrl, - thumbnail = v.thumbnail, - durationSeconds = v.durationSeconds, - viewCount = v.viewCount, - ) + val service = NewPipe.getService(ServiceList.YouTube.serviceId) + val info = withContext(Dispatchers.IO) { + ChannelInfo.getInfo(service, channelUrl) } + // AUD-HIGH: pick the Videos tab specifically rather than + // info.tabs.firstOrNull() which is YouTube's "Home" (a + // curated mix that mostly drops via filterIsInstance). + val videosTab = info.tabs.firstOrNull { + it.contentFilters.contains(ChannelTabs.VIDEOS) + } ?: info.tabs.firstOrNull() + val videos: List = if (videosTab != null) { + withContext(Dispatchers.IO) { + runCatching { + ChannelTabInfo.getInfo(service, videosTab) + .relatedItems + .filterIsInstance() + .map { + StreamItem( + url = it.url, + title = it.name ?: "(no title)", + uploader = it.uploaderName ?: info.name ?: "", + uploaderUrl = it.uploaderUrl ?: channelUrl, + thumbnail = bestThumbnail(it.thumbnails), + durationSeconds = it.duration, + viewCount = it.viewCount, + ) + } + }.getOrDefault(emptyList()) + } + } else emptyList() + _ui.value = ChannelUiState( loading = false, - name = ch.name, - subscriberCount = ch.subscriberCount, - banner = ch.banner, - avatar = ch.avatar, + name = info.name ?: "", + subscriberCount = info.subscriberCount, + banner = bestThumbnail(info.banners), + avatar = bestThumbnail(info.avatars), videos = videos, ) } catch (t: Throwable) { diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt index 54e3ed3e4..33d1c165a 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt @@ -190,7 +190,7 @@ fun VideoDetailScreen( Spacer(modifier = Modifier.height(16.dp)) if (showDownloadDialog) { - val info = state.info // uniffi.strawcore.StreamInfo cached on the UI state + val info = state.streamInfo AlertDialog( onDismissRequest = { showDownloadDialog = false }, title = { Text("Download") }, @@ -208,9 +208,10 @@ fun VideoDetailScreen( confirmButton = { Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { Button(onClick = { - val audio = info?.audioOnly - ?.maxByOrNull { it.bitrate } - ?.url + val audio = info?.audioStreams + ?.filter { it.content?.isNotBlank() == true } + ?.maxByOrNull { it.bitrate ?: 0 } + ?.content if (audio != null) { val id = Downloader.enqueue(context, audio, d.title, DownloadKind.Audio) val msg = if (id > 0) "audio queued" else "download refused (bad URL)" @@ -221,12 +222,14 @@ fun VideoDetailScreen( showDownloadDialog = false }) { Text("Audio") } Button(onClick = { - val video = info?.combined - ?.maxByOrNull { it.bitrate } - ?.url - ?: info?.videoOnly - ?.maxByOrNull { it.bitrate } - ?.url + val video = info?.videoStreams + ?.filter { it.content?.isNotBlank() == true } + ?.maxByOrNull { it.bitrate ?: 0 } + ?.content + ?: info?.videoOnlyStreams + ?.filter { it.content?.isNotBlank() == true } + ?.maxByOrNull { it.bitrate ?: 0 } + ?.content if (video != null) { val id = Downloader.enqueue(context, video, d.title, DownloadKind.Video) val msg = if (id > 0) "video queued" else "download refused (bad URL)" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt index 5ad789aa1..0557c4051 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt @@ -1,11 +1,6 @@ /* * SPDX-FileCopyrightText: 2026 Sulkta-Coop * SPDX-License-Identifier: GPL-3.0-or-later - * - * Phase U-3: extractor moved from NewPipeExtractor (Java) to strawcore - * (Rust + rustypipe), called as a UniFFI suspend fun. The shape of - * VideoDetail and the on-screen behavior are unchanged; only the engine - * underneath flipped. */ package com.sulkta.straw.feature.detail @@ -15,16 +10,18 @@ import androidx.lifecycle.viewModelScope import com.sulkta.straw.data.History import com.sulkta.straw.data.Settings import com.sulkta.straw.data.WatchHistoryItem -import com.sulkta.straw.feature.search.StreamItem import com.sulkta.straw.net.RydClient import com.sulkta.straw.net.RydVotes import com.sulkta.straw.net.SponsorBlockClient +import com.sulkta.straw.util.bestThumbnail import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.stream.StreamInfo +import org.schabi.newpipe.extractor.stream.StreamInfoItem data class VideoDetail( val id: String, @@ -36,15 +33,15 @@ data class VideoDetail( val thumbnail: String?, val ryd: RydVotes? = null, val sbSegmentCount: Int = 0, - val related: List = emptyList(), + val related: List = emptyList(), ) data class VideoDetailUiState( val loading: Boolean = true, val detail: VideoDetail? = null, val error: String? = null, - /** Cached strawcore result so the Player + Download dialog can use it. */ - val info: uniffi.strawcore.StreamInfo? = null, + // Stored on success for handoff to player. Not in UI. + val streamInfo: StreamInfo? = null, ) class VideoDetailViewModel : ViewModel() { @@ -54,18 +51,18 @@ class VideoDetailViewModel : ViewModel() { private var loadedUrl: String? = null fun load(streamUrl: String) { - // Activity-scoped VM is reused across nav entries; only re-fetch when - // the requested URL actually changed. + // viewModel() is Activity-scoped, so the same VM is reused across + // navigations. Compare the requested URL with what we last loaded. if (loadedUrl == streamUrl && _ui.value.detail != null) return loadedUrl = streamUrl _ui.value = VideoDetailUiState(loading = true) viewModelScope.launch { try { - val info = uniffi.strawcore.streamInfo(streamUrl) + val info = withContext(Dispatchers.IO) { StreamInfo.getInfo(streamUrl) } val videoId = info.id - val title = info.title.ifBlank { "(no title)" } - val uploader = info.uploader - val thumb = info.thumbnail + val thumb = bestThumbnail(info.thumbnails) + val title = info.name ?: "(no title)" + val uploader = info.uploaderName ?: "" runCatching { History.get().recordWatch( @@ -80,8 +77,6 @@ class VideoDetailViewModel : ViewModel() { ) } - // RYD + SponsorBlock stay in Kotlin (small JSON HTTP clients, - // no extractor logic). val ryd = withContext(Dispatchers.IO) { runCatching { RydClient.fetch(videoId) }.getOrNull() } @@ -89,18 +84,19 @@ class VideoDetailViewModel : ViewModel() { val sbCount = if (sbCats.isEmpty()) 0 else withContext(Dispatchers.IO) { runCatching { SponsorBlockClient.fetch(videoId, sbCats).size }.getOrDefault(0) } - - val related = info.related.map { r -> - StreamItem( - url = r.url, - title = r.title.ifBlank { "(no title)" }, - uploader = r.uploader, - uploaderUrl = r.uploaderUrl, - thumbnail = r.thumbnail, - durationSeconds = r.durationSeconds, - viewCount = r.viewCount, - ) - } + val related = info.relatedItems + ?.filterIsInstance() + ?.map { it -> + com.sulkta.straw.feature.search.StreamItem( + url = it.url, + title = it.name ?: "(no title)", + uploader = it.uploaderName ?: "", + uploaderUrl = it.uploaderUrl, + thumbnail = bestThumbnail(it.thumbnails), + durationSeconds = it.duration, + viewCount = it.viewCount, + ) + } ?: emptyList() _ui.value = VideoDetailUiState( loading = false, @@ -110,13 +106,13 @@ class VideoDetailViewModel : ViewModel() { uploader = uploader, uploaderUrl = info.uploaderUrl, viewCount = info.viewCount, - description = info.description, + description = info.description?.content ?: "", thumbnail = thumb, ryd = ryd, sbSegmentCount = sbCount, related = related, ), - info = info, + streamInfo = info, ) } catch (t: Throwable) { _ui.value = VideoDetailUiState( diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/feed/SubscriptionFeedViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/feed/SubscriptionFeedViewModel.kt index 51053855c..93ca74467 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/feed/SubscriptionFeedViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/feed/SubscriptionFeedViewModel.kt @@ -2,14 +2,16 @@ * SPDX-FileCopyrightText: 2026 Sulkta-Coop * SPDX-License-Identifier: GPL-3.0-or-later * - * Phase Q + U-4: aggregate latest videos across all subscribed channels. - * Per-channel fetches now go through strawcore (rustypipe + UniFFI). The - * Kotlin side still owns the structured concurrency: - * - cancels prior in-flight refresh - * - Semaphore caps parallel fetches (rustypipe internally has its own - * HTTP pool but we still want to avoid N=100 concurrent extractor - * contexts when N=100 channels) - * - per-channel 15s timeout + * Phase Q: aggregate latest videos across all subscribed channels into a + * single feed. Fans out per-channel ChannelInfo + ChannelTabs.VIDEOS + * fetches in parallel, merges by view count desc, caps at 200 items. + * + * Audit fixes (2026-05-24 pass #2): + * HIGH-6: cancel any prior in-flight refresh when a new one starts, cap + * concurrency with a Semaphore, time-bound each per-channel fetch so + * one hung channel can't stall the whole feed. + * MED-7: use `update { }` for atomic UI-state writes (matches the + * convention applied to the data stores in audit pass #1). */ package com.sulkta.straw.feature.feed @@ -18,7 +20,9 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.sulkta.straw.data.Subscriptions import com.sulkta.straw.feature.search.StreamItem +import com.sulkta.straw.util.bestThumbnail import com.sulkta.straw.util.strawLogW +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -30,7 +34,14 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.channel.ChannelInfo +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo +import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs +import org.schabi.newpipe.extractor.stream.StreamInfoItem data class SubscriptionFeedUiState( val loading: Boolean = false, @@ -43,11 +54,16 @@ class SubscriptionFeedViewModel : ViewModel() { private val _ui = MutableStateFlow(SubscriptionFeedUiState()) val ui: StateFlow = _ui.asStateFlow() + /** Cache feed for 10 min to avoid hammering YT on tab re-entry. */ private val cacheTtlMs = 10L * 60 * 1000 - private val perChannelTimeoutMs = 15_000L - private val parallelism = 8 - private val perChannelMax = 5 + /** Per-channel fetch timeout — slowest channel can't stall the whole batch. */ + private val perChannelTimeoutMs = 15_000L + + /** Cap parallel network fetches even with 100+ subs. */ + private val parallelism = 8 + + /** Live refresh job, so spam-tapping Refresh doesn't fan out racing fetches. */ private var inFlight: Job? = null fun refreshIfStale() { @@ -66,40 +82,55 @@ class SubscriptionFeedViewModel : ViewModel() { _ui.update { it.copy(loading = true, error = null) } inFlight = viewModelScope.launch { try { - val gate = Semaphore(parallelism) - val items = coroutineScope { - val deferreds = channels.map { ch -> - async { - gate.withPermit { - withTimeoutOrNull(perChannelTimeoutMs) { - runCatching { - uniffi.strawcore.channelInfo(ch.url).videos.take(perChannelMax).map { v -> - StreamItem( - url = v.url, - title = v.title.ifBlank { "(no title)" }, - uploader = v.uploader.ifBlank { ch.name }, - uploaderUrl = v.uploaderUrl ?: ch.url, - thumbnail = v.thumbnail, - durationSeconds = v.durationSeconds, - viewCount = v.viewCount, - ) - } - }.onFailure { - strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" } - }.getOrDefault(emptyList()) - } ?: run { - strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" } - emptyList() + val items = withContext(Dispatchers.IO) { + val service = NewPipe.getService(ServiceList.YouTube.serviceId) + val perChannelMax = 5 + val gate = Semaphore(parallelism) + coroutineScope { + val deferreds = channels.map { ch -> + async { + gate.withPermit { + withTimeoutOrNull(perChannelTimeoutMs) { + runCatching { + val info = ChannelInfo.getInfo(service, ch.url) + val tab = info.tabs.firstOrNull { + it.contentFilters.contains(ChannelTabs.VIDEOS) + } ?: info.tabs.firstOrNull() + ?: return@runCatching emptyList() + ChannelTabInfo.getInfo(service, tab) + .relatedItems + .filterIsInstance() + .take(perChannelMax) + .map { si -> + StreamItem( + url = si.url, + title = si.name ?: "(no title)", + uploader = si.uploaderName ?: ch.name, + uploaderUrl = si.uploaderUrl ?: ch.url, + thumbnail = bestThumbnail(si.thumbnails), + durationSeconds = si.duration, + viewCount = si.viewCount, + ) + } + }.onFailure { + strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" } + }.getOrDefault(emptyList()) + } ?: run { + strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" } + emptyList() + } } } } + deferreds.awaitAll() } - deferreds.awaitAll() + .flatten() + // No reliable upload-timestamp from extractor's StreamInfoItem + // in all cases — sort by view count desc as a soft proxy for + // recency-popularity within the recent window. + .sortedByDescending { it.viewCount } + .take(200) } - .flatten() - .sortedByDescending { it.viewCount } - .take(200) - _ui.update { SubscriptionFeedUiState( loading = false, diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt index 349c49ba8..40b4083c0 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt @@ -50,7 +50,7 @@ import androidx.media3.exoplayer.source.DefaultMediaSourceFactory import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService import com.sulkta.straw.StrawActivity -import com.sulkta.straw.net.STRAW_USER_AGENT +import com.sulkta.straw.extractor.NewPipeDownloader @UnstableApi class PlaybackService : MediaSessionService() { @@ -63,7 +63,7 @@ class PlaybackService : MediaSessionService() { ensureChannel() val httpFactory = DefaultHttpDataSource.Factory() - .setUserAgent(STRAW_USER_AGENT) + .setUserAgent(NewPipeDownloader.USER_AGENT) .setAllowCrossProtocolRedirects(true) val mediaSourceFactory = DefaultMediaSourceFactory(this) .setDataSourceFactory(httpFactory) diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt index 5279e8fdc..1478ca857 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt @@ -70,7 +70,7 @@ import androidx.media3.exoplayer.hls.HlsMediaSource import androidx.media3.exoplayer.source.MergingMediaSource import androidx.media3.exoplayer.source.ProgressiveMediaSource import androidx.media3.ui.PlayerView -import com.sulkta.straw.net.STRAW_USER_AGENT +import com.sulkta.straw.extractor.NewPipeDownloader import com.sulkta.straw.net.SbSegment import com.sulkta.straw.util.strawLogI import kotlinx.coroutines.delay @@ -121,28 +121,6 @@ fun PlayerScreen( } } - // Surface playback errors so a 403/404 from googlevideo doesn't show - // as a silent black screen. Captures everything ExoPlayer's renderer - // pipeline raises. - DisposableEffect(exoPlayer) { - val listener = object : Player.Listener { - override fun onPlayerError(error: androidx.media3.common.PlaybackException) { - val msg = buildString { - append("play err ") - append(error.errorCodeName) - append(": ") - append(error.message ?: error.cause?.message ?: "?") - } - com.sulkta.straw.util.strawLogW("StrawPlayer") { "$msg" } - runCatching { - Toast.makeText(context, msg.take(160), Toast.LENGTH_LONG).show() - } - } - } - exoPlayer.addListener(listener) - onDispose { exoPlayer.removeListener(listener) } - } - // PiP setup: on Android 12+ tell the OS this activity can auto-enter // PiP, so when the user presses Home or swipes away the video shrinks // into a floating window instead of pausing/exiting. Aspect ratio is @@ -191,7 +169,7 @@ fun PlayerScreen( LaunchedEffect(resolved) { val r = resolved ?: return@LaunchedEffect val dataSourceFactory = DefaultHttpDataSource.Factory() - .setUserAgent(STRAW_USER_AGENT) + .setUserAgent(NewPipeDownloader.USER_AGENT) .setAllowCrossProtocolRedirects(true) val source = when { diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerViewModel.kt index 9520993b4..ee523f634 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerViewModel.kt @@ -1,16 +1,13 @@ /* * SPDX-FileCopyrightText: 2026 Sulkta-Coop * SPDX-License-Identifier: GPL-3.0-or-later - * - * Phase U-3: extractor moved from NewPipeExtractor (Java) to strawcore - * (Rust + rustypipe via UniFFI). PlayerScreen still calls vm.resolve(url) - * the same way — the engine underneath flipped. */ package com.sulkta.straw.feature.player import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import com.sulkta.straw.data.MaxResolution import com.sulkta.straw.data.Settings import com.sulkta.straw.net.SbSegment import com.sulkta.straw.net.SponsorBlockClient @@ -20,6 +17,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.stream.StreamInfo data class ResolvedPlayback( val title: String, @@ -50,9 +48,8 @@ class PlayerViewModel : ViewModel() { _ui.value = PlayerUiState(loading = true) viewModelScope.launch { try { - val info = uniffi.strawcore.streamInfo(streamUrl) + val info = withContext(Dispatchers.IO) { StreamInfo.getInfo(streamUrl) } val videoId = info.id - val sbCategories = Settings.get().sbCategories.value.map { it.key } val segments = if (sbCategories.isEmpty()) { emptyList() @@ -64,24 +61,32 @@ class PlayerViewModel : ViewModel() { } val maxRes = Settings.get().maxResolution.value.ceiling + fun heightOf(q: String?): Int = + q?.removeSuffix("p")?.takeWhile { it.isDigit() }?.toIntOrNull() ?: 0 - // Audit HIGH-8 carry-over: filter by max resolution but fall - // back to lowest available if the ceiling excludes everything. - fun pickVideo(streams: List): String? { - if (streams.isEmpty()) return null - val filtered = streams.filter { it.height <= maxRes } - val pool = filtered.ifEmpty { streams } - return pool.maxByOrNull { it.bitrate }?.url + // Audit HIGH-8: when no stream is under the resolution ceiling + // (e.g. user picked 144p but the video only has 360p+), fall + // back to the lowest-resolution available instead of returning + // null and showing a black-screen player. + fun pickVideo(streams: List?): String? { + if (streams.isNullOrEmpty()) return null + val withContent = streams.filter { it.content?.isNotBlank() == true } + val filtered = withContent.filter { heightOf(it.getResolution()) <= maxRes } + val pool = filtered.ifEmpty { withContent } + return pool.maxByOrNull { it.bitrate ?: 0 }?.content } - val combined = pickVideo(info.combined) - val videoOnly = pickVideo(info.videoOnly) - val audioOnly = info.audioOnly.maxByOrNull { it.bitrate }?.url + val combined = pickVideo(info.videoStreams) + val videoOnly = pickVideo(info.videoOnlyStreams) + val audioOnly = info.audioStreams + ?.filter { it.content?.isNotBlank() == true } + ?.maxByOrNull { it.bitrate ?: 0 } + ?.content _ui.value = PlayerUiState( loading = false, resolved = ResolvedPlayback( - title = info.title, + title = info.name ?: "", videoUrl = videoOnly, audioUrl = audioOnly, combinedUrl = combined, diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt index 2a305740a..5ef859be4 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/search/SearchViewModel.kt @@ -8,10 +8,17 @@ package com.sulkta.straw.feature.search import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.sulkta.straw.data.History +import com.sulkta.straw.util.bestThumbnail +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.ServiceList +import org.schabi.newpipe.extractor.search.SearchInfo +import org.schabi.newpipe.extractor.stream.StreamInfoItem data class SearchUiState( val query: String = "", @@ -45,23 +52,7 @@ class SearchViewModel : ViewModel() { _ui.value = _ui.value.copy(loading = true, error = null, results = emptyList()) viewModelScope.launch { try { - // Phase U-2: rustypipe via UniFFI. The bindgen-generated - // `search()` is already a suspend fun running on the tokio - // runtime baked into libstrawcore.so — no Dispatchers.IO - // wrapper needed, the JNI call returns to us on the caller - // dispatcher when the future completes. - val rustItems = uniffi.strawcore.search(q) - val items = rustItems.map { r -> - StreamItem( - url = r.url, - title = r.title.ifBlank { "(no title)" }, - uploader = r.uploader, - uploaderUrl = r.uploaderUrl, - thumbnail = r.thumbnail, - durationSeconds = r.durationSeconds, - viewCount = r.viewCount, - ) - } + val items = withContext(Dispatchers.IO) { search(q) } _ui.value = _ui.value.copy(loading = false, results = items) } catch (t: Throwable) { _ui.value = _ui.value.copy( @@ -71,4 +62,23 @@ class SearchViewModel : ViewModel() { } } } + + private fun search(query: String): List { + val service = NewPipe.getService(ServiceList.YouTube.serviceId) + val qh = service.searchQHFactory.fromQuery(query, emptyList(), "") + val info = SearchInfo.getInfo(service, qh) + return info.relatedItems + .filterIsInstance() + .map { + StreamItem( + url = it.url, + title = it.name ?: "(no title)", + uploader = it.uploaderName ?: "", + uploaderUrl = it.uploaderUrl, + thumbnail = bestThumbnail(it.thumbnails), + durationSeconds = it.duration, + viewCount = it.viewCount, + ) + } + } } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt index 01b0598fd..4a3524a61 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt @@ -13,33 +13,9 @@ package com.sulkta.straw.net -import okhttp3.OkHttpClient import okhttp3.ResponseBody import okio.Buffer import java.io.IOException -import java.util.concurrent.TimeUnit - -/** - * Phase U-5: USER_AGENT + shared OkHttpClient that previously lived on - * NewPipeDownloader. After ripping NewPipeExtractor, the RYD + SponsorBlock - * + ExoPlayer HTTP factories still need both. One shared client is fine. - */ -const val STRAW_USER_AGENT: String = - "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36 Straw/1.0" - -@Volatile -private var sharedClient: OkHttpClient? = null - -fun strawHttpClient(): OkHttpClient = - sharedClient ?: synchronized(STRAW_USER_AGENT) { - sharedClient ?: OkHttpClient.Builder() - .connectTimeout(15, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .followRedirects(true) - .followSslRedirects(true) - .build() - .also { sharedClient = it } - } fun ResponseBody.cappedString(maxBytes: Long): String { val cl = contentLength() diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt index 817597596..7c684b4ac 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt @@ -8,6 +8,7 @@ package com.sulkta.straw.net +import com.sulkta.straw.extractor.NewPipeDownloader import com.sulkta.straw.util.strawLogD import com.sulkta.straw.util.strawLogW import kotlinx.serialization.Serializable @@ -33,11 +34,11 @@ object RydClient { strawLogD(TAG) { "fetch start: $videoId → $url" } val req = Request.Builder() .url(url) - .header("User-Agent", STRAW_USER_AGENT) + .header("User-Agent", NewPipeDownloader.USER_AGENT) .header("Accept", "application/json") .build() return runCatching { - strawHttpClient().newCall(req).execute().use { r -> + NewPipeDownloader.client().newCall(req).execute().use { r -> val code = r.code // AUD-HIGH: bounded body read to defend against OOM. val bodyStr = r.body?.cappedString(RYD_MAX_BYTES) ?: "" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt index 9979e58f8..a5bd4b555 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt @@ -8,6 +8,7 @@ package com.sulkta.straw.net +import com.sulkta.straw.extractor.NewPipeDownloader import com.sulkta.straw.util.strawLogD import com.sulkta.straw.util.strawLogW import kotlinx.serialization.Serializable @@ -46,11 +47,11 @@ object SponsorBlockClient { strawLogD(TAG) { "fetch: videoId=$videoId prefix=$prefix url=$urlStr" } val req = Request.Builder() .url(urlStr) - .header("User-Agent", STRAW_USER_AGENT) + .header("User-Agent", NewPipeDownloader.USER_AGENT) .header("Accept", "application/json") .build() return runCatching { - strawHttpClient().newCall(req).execute().use { r -> + NewPipeDownloader.client().newCall(req).execute().use { r -> val code = r.code // AUD-HIGH: bounded body read. val bodyStr = r.body?.cappedString(SB_MAX_BYTES) ?: "" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt b/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt new file mode 100644 index 000000000..6d47e386b --- /dev/null +++ b/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: 2026 Sulkta-Coop + * SPDX-License-Identifier: GPL-3.0-or-later + * + * NewPipeExtractor returns thumbnails as a List with width/height + * fields. Calling .firstOrNull() picks the smallest (the list is sorted + * ascending) — which gave us pixelated thumbnails. This helper picks the + * largest by pixel area instead. + */ + +package com.sulkta.straw.util + +import org.schabi.newpipe.extractor.Image + +fun bestThumbnail(images: List?): String? { + if (images.isNullOrEmpty()) return null + return images + .maxByOrNull { + val w = it.width.takeIf { v -> v > 0 } ?: 0 + val h = it.height.takeIf { v -> v > 0 } ?: 0 + w.toLong() * h.toLong() + } + ?.url +}