Sulkta-OSS/straw
3
0
Fork 0
Code Issues 1 Pull requests 10 Projects Releases Packages Wiki Activity Actions

v0.1.0-W (vc=10): U-4 + U-5 — channels via rustypipe + rip NewPipeExtractor

Browse source 
channel_info(url) UniFFI suspend fn. ChannelViewModel +
SubscriptionFeedViewModel both swap. NewPipeExtractor (Java) is OUT —
zero org.schabi.newpipe classes in the APK now.

Cleanup:
- NewPipeDownloader.kt deleted (was the OkHttp adapter)
- Thumbnails.kt deleted (rustypipe returns full URLs)
- NewPipe.init() dropped from StrawApp.onCreate
- libs.newpipe.extractor removed from build.gradle.kts
- STRAW_USER_AGENT + strawHttpClient() now live in net/Http.kt
- RydClient + SponsorBlockClient + PlayerScreen + PlaybackService all
  read from net/Http.kt instead of the extractor package

rustypipe API quirks beat:
- channel_videos(id) is the right method (channel() doesn't exist)
- ChannelInfo struct = basic metadata; Channel<T> wrapper carries
  name/avatar/banner + .content is the paginator of videos
- description is String (not Option), subscriber_count is Option<u64>

End state: strawApp Kotlin is ~UI + thin glue to strawcore. The Rust
core handles search / streamInfo / channel / channel_videos via UniFFI
suspend fns. Tokio + reqwest + rustls + rquickjs all packed in
libstrawcore.so (~6MB per ABI). APK 40MB total.
This commit is contained in:
Sulkta 2026-05-24 09:11:14 -07:00
parent 89b9837758
commit ab37d534d5
14 changed files with 213 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

16
strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt
View file

@ -1,6 +1,9 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Phase U-5: NewPipeExtractor (Java) torn out. All YT extraction goes
* through strawcore (Rust + rustypipe + UniFFI) now.
*/
package com.sulkta.straw
@ -9,25 +12,16 @@ import android.app.Application
import com.sulkta.straw.data.History
import com.sulkta.straw.data.Settings
import com.sulkta.straw.data.Subscriptions
import com.sulkta.straw.extractor.NewPipeDownloader
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.localization.ContentCountry
import org.schabi.newpipe.extractor.localization.Localization
class StrawApp : Application() {
override fun onCreate() {
super.onCreate()
NewPipe.init(
NewPipeDownloader.init(),
Localization("en", "US"),
ContentCountry("US"),
)
History.init(this)
Settings.init(this)
Subscriptions.init(this)
// Phase U-1: load the strawcore native library and route its logs
// into android logcat under the "strawcore" tag.
// Load strawcore native + route its logs into android logcat under
// the "strawcore" tag.
runCatching {
System.loadLibrary("strawcore")
uniffi.strawcore.initLogging()

96
strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt
View file

@ -1,96 +0,0 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Minimal OkHttp-backed implementation of NewPipeExtractor's Downloader.
* No cookies, no recaptcha handling — anonymous browsing only. Modeled after
* NewPipe's DownloaderImpl but trimmed down for fork scope.
*/
package com.sulkta.straw.extractor
import com.sulkta.straw.net.NEWPIPE_MAX_BYTES
import com.sulkta.straw.net.cappedString
import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
import org.schabi.newpipe.extractor.downloader.Downloader
import org.schabi.newpipe.extractor.downloader.Request
import org.schabi.newpipe.extractor.downloader.Response
import java.io.IOException
import java.util.concurrent.TimeUnit
class NewPipeDownloader private constructor(
private val client: OkHttpClient,
) : Downloader() {
override fun execute(request: Request): Response {
val httpMethod = request.httpMethod()
val url = request.url()
val headers = request.headers()
val data: ByteArray? = request.dataToSend()
val requestBody = data?.toRequestBody(null)
val okBuilder = okhttp3.Request.Builder()
.method(httpMethod, requestBody)
.url(url)
// AUD-HIGH: copy NPE headers BEFORE adding our explicit UA so the
// explicit UA wins; guard against header values containing \r/\n
// which OkHttp's addHeader rejects via IAE (turning a poisoned
// response into an app crash).
headers.forEach { (name, values) ->
if (name.equals("User-Agent", ignoreCase = true)) return@forEach
okBuilder.removeHeader(name)
values.forEach { value ->
runCatching { okBuilder.addHeader(name, value) }
}
}
okBuilder.removeHeader("User-Agent")
okBuilder.addHeader("User-Agent", USER_AGENT)
val okResponse = client.newCall(okBuilder.build()).execute()
val body = okResponse.body
// AUD-HIGH: bounded read to defend against OOM via gigabyte response.
val bodyString = body?.cappedString(NEWPIPE_MAX_BYTES) ?: ""
val responseHeaders = okResponse.headers.toMultimap()
val latestUrl = okResponse.request.url.toString()
if (okResponse.code == 429) {
okResponse.close()
throw IOException("HTTP 429 — rate limited")
}
okResponse.close()
return Response(
okResponse.code,
okResponse.message,
responseHeaders,
bodyString,
latestUrl,
)
}
companion object {
const val USER_AGENT =
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/120.0.0.0 Mobile Safari/537.36"
@Volatile private var instance: NewPipeDownloader? = null
fun init(builder: OkHttpClient.Builder? = null): NewPipeDownloader {
val client = (builder ?: OkHttpClient.Builder())
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.writeTimeout(30, TimeUnit.SECONDS)
.build()
val d = NewPipeDownloader(client)
instance = d
return d
}
fun get(): NewPipeDownloader = instance
?: error("NewPipeDownloader not initialized — call init() first")
fun client(): OkHttpClient = get().client
}
}

62
strawApp/src/main/kotlin/com/sulkta/straw/feature/channel/ChannelViewModel.kt
View file

@ -1,6 +1,10 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Phase U-4: ChannelInfo + Videos tab moved to strawcore (rustypipe).
* The two separate ChannelInfo.getInfo + ChannelTabInfo.getInfo calls
* collapse into one Rust round-trip.
*/
package com.sulkta.straw.feature.channel
@ -8,19 +12,10 @@ package com.sulkta.straw.feature.channel
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import com.sulkta.straw.feature.search.StreamItem
import com.sulkta.straw.util.bestThumbnail
import kotlinx.coroutines.Dispatchers
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.ServiceList
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import org.schabi.newpipe.extractor.channel.ChannelInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
import org.schabi.newpipe.extractor.stream.StreamInfoItem
data class ChannelUiState(
val loading: Boolean = true,
@ -40,43 +35,24 @@ class ChannelViewModel : ViewModel() {
_ui.value = ChannelUiState(loading = true)
viewModelScope.launch {
try {
val service = NewPipe.getService(ServiceList.YouTube.serviceId)
val info = withContext(Dispatchers.IO) {
ChannelInfo.getInfo(service, channelUrl)
val ch = uniffi.strawcore.channelInfo(channelUrl)
val videos = ch.videos.map { v ->
StreamItem(
url = v.url,
title = v.title.ifBlank { "(no title)" },
uploader = v.uploader,
uploaderUrl = v.uploaderUrl,
thumbnail = v.thumbnail,
durationSeconds = v.durationSeconds,
viewCount = v.viewCount,
)
}
// AUD-HIGH: pick the Videos tab specifically rather than
// info.tabs.firstOrNull() which is YouTube's "Home" (a
// curated mix that mostly drops via filterIsInstance).
val videosTab = info.tabs.firstOrNull {
it.contentFilters.contains(ChannelTabs.VIDEOS)
} ?: info.tabs.firstOrNull()
val videos: List<StreamItem> = if (videosTab != null) {
withContext(Dispatchers.IO) {
runCatching {
ChannelTabInfo.getInfo(service, videosTab)
.relatedItems
.filterIsInstance<StreamInfoItem>()
.map {
StreamItem(
url = it.url,
title = it.name ?: "(no title)",
uploader = it.uploaderName ?: info.name ?: "",
uploaderUrl = it.uploaderUrl ?: channelUrl,
thumbnail = bestThumbnail(it.thumbnails),
durationSeconds = it.duration,
viewCount = it.viewCount,
)
}
}.getOrDefault(emptyList())
}
} else emptyList()
_ui.value = ChannelUiState(
loading = false,
name = info.name ?: "",
subscriberCount = info.subscriberCount,
banner = bestThumbnail(info.banners),
avatar = bestThumbnail(info.avatars),
name = ch.name,
subscriberCount = ch.subscriberCount,
banner = ch.banner,
avatar = ch.avatar,
videos = videos,
)
} catch (t: Throwable) {

107
strawApp/src/main/kotlin/com/sulkta/straw/feature/feed/SubscriptionFeedViewModel.kt
View file

@ -2,16 +2,14 @@
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Phase Q: aggregate latest videos across all subscribed channels into a
* single feed. Fans out per-channel ChannelInfo + ChannelTabs.VIDEOS
* fetches in parallel, merges by view count desc, caps at 200 items.
*
* Audit fixes (2026-05-24 pass #2):
* HIGH-6: cancel any prior in-flight refresh when a new one starts, cap
* concurrency with a Semaphore, time-bound each per-channel fetch so
* one hung channel can't stall the whole feed.
* MED-7: use `update { }` for atomic UI-state writes (matches the
* convention applied to the data stores in audit pass #1).
* Phase Q + U-4: aggregate latest videos across all subscribed channels.
* Per-channel fetches now go through strawcore (rustypipe + UniFFI). The
* Kotlin side still owns the structured concurrency:
* - cancels prior in-flight refresh
* - Semaphore caps parallel fetches (rustypipe internally has its own
* HTTP pool but we still want to avoid N=100 concurrent extractor
* contexts when N=100 channels)
* - per-channel 15s timeout
*/
package com.sulkta.straw.feature.feed
@ -20,9 +18,7 @@ import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import com.sulkta.straw.data.Subscriptions
import com.sulkta.straw.feature.search.StreamItem
import com.sulkta.straw.util.bestThumbnail
import com.sulkta.straw.util.strawLogW
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
@ -34,14 +30,7 @@ import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.ServiceList
import org.schabi.newpipe.extractor.channel.ChannelInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
import org.schabi.newpipe.extractor.stream.StreamInfoItem
data class SubscriptionFeedUiState(
val loading: Boolean = false,
@ -54,16 +43,11 @@ class SubscriptionFeedViewModel : ViewModel() {
private val _ui = MutableStateFlow(SubscriptionFeedUiState())
val ui: StateFlow<SubscriptionFeedUiState> = _ui.asStateFlow()
/** Cache feed for 10 min to avoid hammering YT on tab re-entry. */
private val cacheTtlMs = 10L * 60 * 1000
/** Per-channel fetch timeout — slowest channel can't stall the whole batch. */
private val perChannelTimeoutMs = 15_000L
/** Cap parallel network fetches even with 100+ subs. */
private val parallelism = 8
private val perChannelMax = 5
/** Live refresh job, so spam-tapping Refresh doesn't fan out racing fetches. */
private var inFlight: Job? = null
fun refreshIfStale() {
@ -82,55 +66,40 @@ class SubscriptionFeedViewModel : ViewModel() {
_ui.update { it.copy(loading = true, error = null) }
inFlight = viewModelScope.launch {
try {
val items = withContext(Dispatchers.IO) {
val service = NewPipe.getService(ServiceList.YouTube.serviceId)
val perChannelMax = 5
val gate = Semaphore(parallelism)
coroutineScope {
val deferreds = channels.map { ch ->
async {
gate.withPermit {
withTimeoutOrNull(perChannelTimeoutMs) {
runCatching {
val info = ChannelInfo.getInfo(service, ch.url)
val tab = info.tabs.firstOrNull {
it.contentFilters.contains(ChannelTabs.VIDEOS)
} ?: info.tabs.firstOrNull()
?: return@runCatching emptyList<StreamItem>()
ChannelTabInfo.getInfo(service, tab)
.relatedItems
.filterIsInstance<StreamInfoItem>()
.take(perChannelMax)
.map { si ->
StreamItem(
url = si.url,
title = si.name ?: "(no title)",
uploader = si.uploaderName ?: ch.name,
uploaderUrl = si.uploaderUrl ?: ch.url,
thumbnail = bestThumbnail(si.thumbnails),
durationSeconds = si.duration,
viewCount = si.viewCount,
)
}
}.onFailure {
strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" }
}.getOrDefault(emptyList())
} ?: run {
strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" }
emptyList()
}
val gate = Semaphore(parallelism)
val items = coroutineScope {
val deferreds = channels.map { ch ->
async {
gate.withPermit {
withTimeoutOrNull(perChannelTimeoutMs) {
runCatching {
uniffi.strawcore.channelInfo(ch.url).videos.take(perChannelMax).map { v ->
StreamItem(
url = v.url,
title = v.title.ifBlank { "(no title)" },
uploader = v.uploader.ifBlank { ch.name },
uploaderUrl = v.uploaderUrl ?: ch.url,
thumbnail = v.thumbnail,
durationSeconds = v.durationSeconds,
viewCount = v.viewCount,
)
}
}.onFailure {
strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" }
}.getOrDefault(emptyList())
} ?: run {
strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" }
emptyList()
}
}
}
deferreds.awaitAll()
}
.flatten()
// No reliable upload-timestamp from extractor's StreamInfoItem
// in all cases — sort by view count desc as a soft proxy for
// recency-popularity within the recent window.
.sortedByDescending { it.viewCount }
.take(200)
deferreds.awaitAll()
}
.flatten()
.sortedByDescending { it.viewCount }
.take(200)
_ui.update {
SubscriptionFeedUiState(
loading = false,

4
strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt
View file

@ -50,7 +50,7 @@ import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
import androidx.media3.session.MediaSession
import androidx.media3.session.MediaSessionService
import com.sulkta.straw.StrawActivity
import com.sulkta.straw.extractor.NewPipeDownloader
import com.sulkta.straw.net.STRAW_USER_AGENT
@UnstableApi
class PlaybackService : MediaSessionService() {
@ -63,7 +63,7 @@ class PlaybackService : MediaSessionService() {
ensureChannel()
val httpFactory = DefaultHttpDataSource.Factory()
.setUserAgent(NewPipeDownloader.USER_AGENT)
.setUserAgent(STRAW_USER_AGENT)
.setAllowCrossProtocolRedirects(true)
val mediaSourceFactory = DefaultMediaSourceFactory(this)
.setDataSourceFactory(httpFactory)

4
strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt
View file

@ -70,7 +70,7 @@ import androidx.media3.exoplayer.hls.HlsMediaSource
import androidx.media3.exoplayer.source.MergingMediaSource
import androidx.media3.exoplayer.source.ProgressiveMediaSource
import androidx.media3.ui.PlayerView
import com.sulkta.straw.extractor.NewPipeDownloader
import com.sulkta.straw.net.STRAW_USER_AGENT
import com.sulkta.straw.net.SbSegment
import com.sulkta.straw.util.strawLogI
import kotlinx.coroutines.delay
@ -169,7 +169,7 @@ fun PlayerScreen(
LaunchedEffect(resolved) {
val r = resolved ?: return@LaunchedEffect
val dataSourceFactory = DefaultHttpDataSource.Factory()
.setUserAgent(NewPipeDownloader.USER_AGENT)
.setUserAgent(STRAW_USER_AGENT)
.setAllowCrossProtocolRedirects(true)
val source = when {

24
strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt
View file

@ -13,9 +13,33 @@
package com.sulkta.straw.net
import okhttp3.OkHttpClient
import okhttp3.ResponseBody
import okio.Buffer
import java.io.IOException
import java.util.concurrent.TimeUnit
/**
* Phase U-5: USER_AGENT + shared OkHttpClient that previously lived on
* NewPipeDownloader. After ripping NewPipeExtractor, the RYD + SponsorBlock
* + ExoPlayer HTTP factories still need both. One shared client is fine.
*/
const val STRAW_USER_AGENT: String =
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36 Straw/1.0"
@Volatile
private var sharedClient: OkHttpClient? = null
fun strawHttpClient(): OkHttpClient =
sharedClient ?: synchronized(STRAW_USER_AGENT) {
sharedClient ?: OkHttpClient.Builder()
.connectTimeout(15, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.followRedirects(true)
.followSslRedirects(true)
.build()
.also { sharedClient = it }
}
fun ResponseBody.cappedString(maxBytes: Long): String {
val cl = contentLength()

5
strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt
View file

@ -8,7 +8,6 @@
package com.sulkta.straw.net
import com.sulkta.straw.extractor.NewPipeDownloader
import com.sulkta.straw.util.strawLogD
import com.sulkta.straw.util.strawLogW
import kotlinx.serialization.Serializable
@ -34,11 +33,11 @@ object RydClient {
strawLogD(TAG) { "fetch start: $videoId → $url" }
val req = Request.Builder()
.url(url)
.header("User-Agent", NewPipeDownloader.USER_AGENT)
.header("User-Agent", STRAW_USER_AGENT)
.header("Accept", "application/json")
.build()
return runCatching {
NewPipeDownloader.client().newCall(req).execute().use { r ->
strawHttpClient().newCall(req).execute().use { r ->
val code = r.code
// AUD-HIGH: bounded body read to defend against OOM.
val bodyStr = r.body?.cappedString(RYD_MAX_BYTES) ?: ""

5
strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt
View file

@ -8,7 +8,6 @@
package com.sulkta.straw.net
import com.sulkta.straw.extractor.NewPipeDownloader
import com.sulkta.straw.util.strawLogD
import com.sulkta.straw.util.strawLogW
import kotlinx.serialization.Serializable
@ -47,11 +46,11 @@ object SponsorBlockClient {
strawLogD(TAG) { "fetch: videoId=$videoId prefix=$prefix url=$urlStr" }
val req = Request.Builder()
.url(urlStr)
.header("User-Agent", NewPipeDownloader.USER_AGENT)
.header("User-Agent", STRAW_USER_AGENT)
.header("Accept", "application/json")
.build()
return runCatching {
NewPipeDownloader.client().newCall(req).execute().use { r ->
strawHttpClient().newCall(req).execute().use { r ->
val code = r.code
// AUD-HIGH: bounded body read.
val bodyStr = r.body?.cappedString(SB_MAX_BYTES) ?: ""

24
strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt
View file

@ -1,24 +0,0 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* NewPipeExtractor returns thumbnails as a List<Image> with width/height
* fields. Calling .firstOrNull() picks the smallest (the list is sorted
* ascending) — which gave us pixelated thumbnails. This helper picks the
* largest by pixel area instead.
*/
package com.sulkta.straw.util
import org.schabi.newpipe.extractor.Image
fun bestThumbnail(images: List<Image>?): String? {
if (images.isNullOrEmpty()) return null
return images
.maxByOrNull {
val w = it.width.takeIf { v -> v > 0 } ?: 0
val h = it.height.takeIf { v -> v > 0 } ?: 0
w.toLong() * h.toLong()
}
?.url
}