From 979b4021b09469735a0bfaddca41224256a9c2fb Mon Sep 17 00:00:00 2001 From: Kayos Date: Sun, 24 May 2026 13:29:19 -0700 Subject: [PATCH] Path C-6: rip NewPipeExtractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero org.schabi.newpipe classes in straw Kotlin. strawcore (Rust + rustypipe via UniFFI) is the only extractor. Deletions: - strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt (was the OkHttp adapter; STRAW_USER_AGENT + strawHttpClient() in net/Http.kt cover its role) - strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt (rustypipe surfaces a pre-picked thumbnail URL; no helper needed) Build: - libs.newpipe.extractor dependency removed from strawApp/build.gradle.kts Call-site swaps: - net/RydClient.kt, net/SponsorBlockClient.kt: NewPipeDownloader.client() + .USER_AGENT → strawHttpClient() + STRAW_USER_AGENT - feature/player/PlayerScreen.kt, feature/player/PlaybackService.kt, feature/detail/VideoDetailScreen.kt: ExoPlayer DefaultHttpDataSource.Factory now reads STRAW_USER_AGENT from net/Http - StrawApp.kt: NewPipe.init() call gone; History/Settings/Subscriptions bootstrap unchanged net/Http.kt gains STRAW_USER_AGENT const + strawHttpClient() lazy OkHttpClient with the same shape NewPipeDownloader had (15s connect, 30s read, follow redirects). --- strawApp/build.gradle.kts | 4 +- .../main/kotlin/com/sulkta/straw/StrawApp.kt | 12 +-- .../straw/extractor/NewPipeDownloader.kt | 96 ------------------- .../straw/feature/detail/VideoDetailScreen.kt | 4 +- .../straw/feature/player/PlaybackService.kt | 4 +- .../straw/feature/player/PlayerScreen.kt | 4 +- .../main/kotlin/com/sulkta/straw/net/Http.kt | 25 +++++ .../kotlin/com/sulkta/straw/net/RydClient.kt | 5 +- .../sulkta/straw/net/SponsorBlockClient.kt | 5 +- .../com/sulkta/straw/util/Thumbnails.kt | 24 ----- 10 files changed, 41 insertions(+), 142 deletions(-) delete mode 100644 strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt delete mode 100644 strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt diff --git a/strawApp/build.gradle.kts b/strawApp/build.gradle.kts index 0a4c50df7..774719420 100644 --- a/strawApp/build.gradle.kts +++ b/strawApp/build.gradle.kts @@ -95,7 +95,9 @@ dependencies { implementation(libs.coil.network.okhttp) // NewPipeExtractor (JVM/Android-only) + its OkHttp dep - implementation(libs.newpipe.extractor) + // libs.newpipe.extractor — REMOVED in Path C-6. Extractor is now strawcore + // (Rust + rustypipe via UniFFI). See rust/strawcore/ + the cargoBuild + + // uniffiBindgen Gradle tasks below. implementation(libs.squareup.okhttp) // JSON for SponsorBlock + Return YouTube Dislike clients diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt index 6d8a1defc..38402f821 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt @@ -9,19 +9,13 @@ import android.app.Application import com.sulkta.straw.data.History import com.sulkta.straw.data.Settings import com.sulkta.straw.data.Subscriptions -import com.sulkta.straw.extractor.NewPipeDownloader -import org.schabi.newpipe.extractor.NewPipe -import org.schabi.newpipe.extractor.localization.ContentCountry -import org.schabi.newpipe.extractor.localization.Localization class StrawApp : Application() { override fun onCreate() { super.onCreate() - NewPipe.init( - NewPipeDownloader.init(), - Localization("en", "US"), - ContentCountry("US"), - ) + // Path C-6 / Phase U-5: NewPipeExtractor is out. strawcore (Rust) + // loads its own libstrawcore.so via JNA when first called — no + // explicit init needed here. Just bootstrap the local stores. History.init(this) Settings.init(this) Subscriptions.init(this) diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt b/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt deleted file mode 100644 index bc185b5e0..000000000 --- a/strawApp/src/main/kotlin/com/sulkta/straw/extractor/NewPipeDownloader.kt +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2026 Sulkta-Coop - * SPDX-License-Identifier: GPL-3.0-or-later - * - * Minimal OkHttp-backed implementation of NewPipeExtractor's Downloader. - * No cookies, no recaptcha handling — anonymous browsing only. Modeled after - * NewPipe's DownloaderImpl but trimmed down for fork scope. - */ - -package com.sulkta.straw.extractor - -import com.sulkta.straw.net.NEWPIPE_MAX_BYTES -import com.sulkta.straw.net.cappedString -import okhttp3.OkHttpClient -import okhttp3.RequestBody.Companion.toRequestBody -import org.schabi.newpipe.extractor.downloader.Downloader -import org.schabi.newpipe.extractor.downloader.Request -import org.schabi.newpipe.extractor.downloader.Response -import java.io.IOException -import java.util.concurrent.TimeUnit - -class NewPipeDownloader private constructor( - private val client: OkHttpClient, -) : Downloader() { - - override fun execute(request: Request): Response { - val httpMethod = request.httpMethod() - val url = request.url() - val headers = request.headers() - val data: ByteArray? = request.dataToSend() - - val requestBody = data?.toRequestBody(null) - - val okBuilder = okhttp3.Request.Builder() - .method(httpMethod, requestBody) - .url(url) - - // AUD-HIGH: copy NPE headers BEFORE adding our explicit UA so the - // explicit UA wins; guard against header values containing \r/\n - // which OkHttp's addHeader rejects via IAE (turning a poisoned - // response into an app crash). - headers.forEach { (name, values) -> - if (name.equals("User-Agent", ignoreCase = true)) return@forEach - okBuilder.removeHeader(name) - values.forEach { value -> - runCatching { okBuilder.addHeader(name, value) } - } - } - okBuilder.removeHeader("User-Agent") - okBuilder.addHeader("User-Agent", USER_AGENT) - - val okResponse = client.newCall(okBuilder.build()).execute() - val body = okResponse.body - // AUD-HIGH: bounded read to defend against OOM via gigabyte response. - val bodyString = body?.cappedString(NEWPIPE_MAX_BYTES) ?: "" - val responseHeaders = okResponse.headers.toMultimap() - val latestUrl = okResponse.request.url.toString() - if (okResponse.code == 429) { - okResponse.close() - throw IOException("HTTP 429 — rate limited") - } - okResponse.close() - - return Response( - okResponse.code, - okResponse.message, - responseHeaders, - bodyString, - latestUrl, - ) - } - - companion object { - const val USER_AGENT = - "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " + - "Chrome/120.0.0.0 Mobile Safari/537.36" - - @Volatile private var instance: NewPipeDownloader? = null - - fun init(builder: OkHttpClient.Builder? = null): NewPipeDownloader { - val client = (builder ?: OkHttpClient.Builder()) - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .build() - val d = NewPipeDownloader(client) - instance = d - return d - } - - fun get(): NewPipeDownloader = instance - ?: error("NewPipeDownloader not initialized — call init() first") - - fun client(): OkHttpClient = get().client - } -} diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt index 679be2de1..1bd6cae8c 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailScreen.kt @@ -69,7 +69,7 @@ import androidx.media3.exoplayer.source.MergingMediaSource import androidx.media3.exoplayer.source.ProgressiveMediaSource import androidx.media3.ui.PlayerView import coil3.compose.AsyncImage -import com.sulkta.straw.extractor.NewPipeDownloader +import com.sulkta.straw.net.STRAW_USER_AGENT import com.sulkta.straw.util.formatCount import com.sulkta.straw.util.formatViews import com.sulkta.straw.util.stripHtml @@ -409,7 +409,7 @@ private fun InlinePlayer( LaunchedEffect(resolved) { val r = resolved ?: return@LaunchedEffect val dataSourceFactory = DefaultHttpDataSource.Factory() - .setUserAgent(NewPipeDownloader.USER_AGENT) + .setUserAgent(STRAW_USER_AGENT) .setAllowCrossProtocolRedirects(true) val source = when { r.dashMpdUrl != null -> DashMediaSource.Factory(dataSourceFactory) diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt index 40b4083c0..349c49ba8 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt @@ -50,7 +50,7 @@ import androidx.media3.exoplayer.source.DefaultMediaSourceFactory import androidx.media3.session.MediaSession import androidx.media3.session.MediaSessionService import com.sulkta.straw.StrawActivity -import com.sulkta.straw.extractor.NewPipeDownloader +import com.sulkta.straw.net.STRAW_USER_AGENT @UnstableApi class PlaybackService : MediaSessionService() { @@ -63,7 +63,7 @@ class PlaybackService : MediaSessionService() { ensureChannel() val httpFactory = DefaultHttpDataSource.Factory() - .setUserAgent(NewPipeDownloader.USER_AGENT) + .setUserAgent(STRAW_USER_AGENT) .setAllowCrossProtocolRedirects(true) val mediaSourceFactory = DefaultMediaSourceFactory(this) .setDataSourceFactory(httpFactory) diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt index 1478ca857..dfb0518e0 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlayerScreen.kt @@ -70,7 +70,7 @@ import androidx.media3.exoplayer.hls.HlsMediaSource import androidx.media3.exoplayer.source.MergingMediaSource import androidx.media3.exoplayer.source.ProgressiveMediaSource import androidx.media3.ui.PlayerView -import com.sulkta.straw.extractor.NewPipeDownloader +import com.sulkta.straw.net.STRAW_USER_AGENT import com.sulkta.straw.net.SbSegment import com.sulkta.straw.util.strawLogI import kotlinx.coroutines.delay @@ -169,7 +169,7 @@ fun PlayerScreen( LaunchedEffect(resolved) { val r = resolved ?: return@LaunchedEffect val dataSourceFactory = DefaultHttpDataSource.Factory() - .setUserAgent(NewPipeDownloader.USER_AGENT) + .setUserAgent(STRAW_USER_AGENT) .setAllowCrossProtocolRedirects(true) val source = when { diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt index 4a3524a61..a8d629d86 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/Http.kt @@ -13,9 +13,34 @@ package com.sulkta.straw.net +import okhttp3.OkHttpClient import okhttp3.ResponseBody import okio.Buffer import java.io.IOException +import java.util.concurrent.TimeUnit + +/** + * Path C-6 / Phase U-5: USER_AGENT + shared OkHttpClient that previously + * lived on NewPipeDownloader. After ripping NewPipeExtractor, the RYD + + * SponsorBlock + ExoPlayer HTTP factories still need both. One shared + * client is fine. + */ +const val STRAW_USER_AGENT: String = + "Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36 Straw/1.0" + +@Volatile +private var sharedClient: OkHttpClient? = null + +fun strawHttpClient(): OkHttpClient = + sharedClient ?: synchronized(STRAW_USER_AGENT) { + sharedClient ?: OkHttpClient.Builder() + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .followRedirects(true) + .followSslRedirects(true) + .build() + .also { sharedClient = it } + } fun ResponseBody.cappedString(maxBytes: Long): String { val cl = contentLength() diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt index 7c684b4ac..817597596 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt @@ -8,7 +8,6 @@ package com.sulkta.straw.net -import com.sulkta.straw.extractor.NewPipeDownloader import com.sulkta.straw.util.strawLogD import com.sulkta.straw.util.strawLogW import kotlinx.serialization.Serializable @@ -34,11 +33,11 @@ object RydClient { strawLogD(TAG) { "fetch start: $videoId → $url" } val req = Request.Builder() .url(url) - .header("User-Agent", NewPipeDownloader.USER_AGENT) + .header("User-Agent", STRAW_USER_AGENT) .header("Accept", "application/json") .build() return runCatching { - NewPipeDownloader.client().newCall(req).execute().use { r -> + strawHttpClient().newCall(req).execute().use { r -> val code = r.code // AUD-HIGH: bounded body read to defend against OOM. val bodyStr = r.body?.cappedString(RYD_MAX_BYTES) ?: "" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt index a5bd4b555..9979e58f8 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt @@ -8,7 +8,6 @@ package com.sulkta.straw.net -import com.sulkta.straw.extractor.NewPipeDownloader import com.sulkta.straw.util.strawLogD import com.sulkta.straw.util.strawLogW import kotlinx.serialization.Serializable @@ -47,11 +46,11 @@ object SponsorBlockClient { strawLogD(TAG) { "fetch: videoId=$videoId prefix=$prefix url=$urlStr" } val req = Request.Builder() .url(urlStr) - .header("User-Agent", NewPipeDownloader.USER_AGENT) + .header("User-Agent", STRAW_USER_AGENT) .header("Accept", "application/json") .build() return runCatching { - NewPipeDownloader.client().newCall(req).execute().use { r -> + strawHttpClient().newCall(req).execute().use { r -> val code = r.code // AUD-HIGH: bounded body read. val bodyStr = r.body?.cappedString(SB_MAX_BYTES) ?: "" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt b/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt deleted file mode 100644 index 6d47e386b..000000000 --- a/strawApp/src/main/kotlin/com/sulkta/straw/util/Thumbnails.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2026 Sulkta-Coop - * SPDX-License-Identifier: GPL-3.0-or-later - * - * NewPipeExtractor returns thumbnails as a List with width/height - * fields. Calling .firstOrNull() picks the smallest (the list is sorted - * ascending) — which gave us pixelated thumbnails. This helper picks the - * largest by pixel area instead. - */ - -package com.sulkta.straw.util - -import org.schabi.newpipe.extractor.Image - -fun bestThumbnail(images: List?): String? { - if (images.isNullOrEmpty()) return null - return images - .maxByOrNull { - val w = it.width.takeIf { v -> v > 0 } ?: 0 - val h = it.height.takeIf { v -> v > 0 } ?: 0 - w.toLong() * h.toLong() - } - ?.url -}