v0.1.0-W (vc=10): U-4 + U-5 — channels via rustypipe + rip NewPipeExtractor
channel_info(url) UniFFI suspend fn. ChannelViewModel + SubscriptionFeedViewModel both swap. NewPipeExtractor (Java) is OUT — zero org.schabi.newpipe classes in the APK now. Cleanup: - NewPipeDownloader.kt deleted (was the OkHttp adapter) - Thumbnails.kt deleted (rustypipe returns full URLs) - NewPipe.init() dropped from StrawApp.onCreate - libs.newpipe.extractor removed from build.gradle.kts - STRAW_USER_AGENT + strawHttpClient() now live in net/Http.kt - RydClient + SponsorBlockClient + PlayerScreen + PlaybackService all read from net/Http.kt instead of the extractor package rustypipe API quirks beat: - channel_videos(id) is the right method (channel() doesn't exist) - ChannelInfo struct = basic metadata; Channel<T> wrapper carries name/avatar/banner + .content is the paginator of videos - description is String (not Option), subscriber_count is Option<u64> End state: strawApp Kotlin is ~UI + thin glue to strawcore. The Rust core handles search / streamInfo / channel / channel_videos via UniFFI suspend fns. Tokio + reqwest + rustls + rquickjs all packed in libstrawcore.so (~6MB per ABI). APK 40MB total.
This commit is contained in:
parent
89b9837758
commit
ab37d534d5
14 changed files with 213 additions and 257 deletions
|
|
@ -15,6 +15,6 @@ const val NEWPIPE_APPLICATION_ID_OLD = "org.schabi.newpipe"
|
||||||
const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app"
|
const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app"
|
||||||
|
|
||||||
// Sulkta fork — Straw
|
// Sulkta fork — Straw
|
||||||
const val STRAW_VERSION_CODE = 9
|
const val STRAW_VERSION_CODE = 10
|
||||||
const val STRAW_VERSION_NAME = "0.1.0-V"
|
const val STRAW_VERSION_NAME = "0.1.0-W"
|
||||||
const val STRAW_APPLICATION_ID = "com.sulkta.straw"
|
const val STRAW_APPLICATION_ID = "com.sulkta.straw"
|
||||||
|
|
|
||||||
113
rust/strawcore/src/channel.rs
Normal file
113
rust/strawcore/src/channel.rs
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
// Phase U-4 — `channel_info(channel_url)` via rustypipe.
|
||||||
|
//
|
||||||
|
// Returns channel metadata + the channel's latest videos (the "Videos" tab).
|
||||||
|
// Used by ChannelScreen (single-channel view) AND
|
||||||
|
// SubscriptionFeedViewModel (which fans out across all subscriptions).
|
||||||
|
|
||||||
|
use crate::error::StrawcoreError;
|
||||||
|
use crate::search::SearchItem;
|
||||||
|
use rustypipe::client::RustyPipe;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, uniffi::Record)]
|
||||||
|
pub struct ChannelInfo {
|
||||||
|
pub id: String,
|
||||||
|
pub name: String,
|
||||||
|
pub avatar: Option<String>,
|
||||||
|
pub banner: Option<String>,
|
||||||
|
/// -1 = unknown / hidden by the channel.
|
||||||
|
pub subscriber_count: i64,
|
||||||
|
pub description: String,
|
||||||
|
/// Latest videos from the channel (Videos tab, newest first).
|
||||||
|
pub videos: Vec<SearchItem>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn yt_video_url(id: &str) -> String {
|
||||||
|
format!("https://www.youtube.com/watch?v={}", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn yt_channel_url(id: &str) -> String {
|
||||||
|
format!("https://www.youtube.com/channel/{}", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Channel-id extraction. Accepts:
|
||||||
|
/// https://www.youtube.com/channel/UC...
|
||||||
|
/// https://www.youtube.com/@handle
|
||||||
|
/// https://www.youtube.com/c/handle
|
||||||
|
/// https://www.youtube.com/user/handle
|
||||||
|
/// bare channel id (UC..., 24 chars)
|
||||||
|
fn extract_channel_input(input: &str) -> Result<String, StrawcoreError> {
|
||||||
|
let trimmed = input.trim();
|
||||||
|
// Bare channel ID — usually 24 chars starting with UC.
|
||||||
|
if trimmed.starts_with("UC") && trimmed.len() == 24 {
|
||||||
|
return Ok(trimmed.to_string());
|
||||||
|
}
|
||||||
|
let url = url::Url::parse(trimmed).map_err(|e| StrawcoreError::Unsupported {
|
||||||
|
detail: format!("bad URL: {}", e),
|
||||||
|
})?;
|
||||||
|
let path = url.path().trim_start_matches('/').trim_end_matches('/');
|
||||||
|
// /channel/UCxxx — canonical
|
||||||
|
if let Some(rest) = path.strip_prefix("channel/") {
|
||||||
|
let id = rest.split('/').next().unwrap_or("");
|
||||||
|
if !id.is_empty() {
|
||||||
|
return Ok(id.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// /@handle — rustypipe takes the handle (with @)
|
||||||
|
if path.starts_with('@') {
|
||||||
|
return Ok(path.split('/').next().unwrap_or(path).to_string());
|
||||||
|
}
|
||||||
|
// /c/name or /user/name
|
||||||
|
for prefix in ["c/", "user/"] {
|
||||||
|
if let Some(rest) = path.strip_prefix(prefix) {
|
||||||
|
let name = rest.split('/').next().unwrap_or("");
|
||||||
|
if !name.is_empty() {
|
||||||
|
// Rustypipe channel() takes the channel id or @handle. For
|
||||||
|
// legacy /c/ and /user/ URLs we prepend @ as a best-effort.
|
||||||
|
return Ok(format!("@{}", name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(StrawcoreError::Unsupported {
|
||||||
|
detail: format!("unsupported channel URL: {}", input),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[uniffi::export(async_runtime = "tokio")]
|
||||||
|
pub async fn channel_info(channel_url: String) -> Result<ChannelInfo, StrawcoreError> {
|
||||||
|
let key = extract_channel_input(&channel_url)?;
|
||||||
|
log::info!("strawcore::channel_info key={}", key);
|
||||||
|
let rp = RustyPipe::new();
|
||||||
|
|
||||||
|
// channel_videos(id) returns Channel<Paginator<VideoItem>> — the
|
||||||
|
// Channel<T> wrapper carries name/avatar/banner/etc and `.content`
|
||||||
|
// is the paginator of videos. One round-trip gets us everything.
|
||||||
|
let channel = rp.query().channel_videos(&key).await?;
|
||||||
|
|
||||||
|
let videos: Vec<SearchItem> = channel
|
||||||
|
.content
|
||||||
|
.items
|
||||||
|
.into_iter()
|
||||||
|
.map(|v| SearchItem {
|
||||||
|
url: yt_video_url(&v.id),
|
||||||
|
title: v.name.clone(),
|
||||||
|
uploader: channel.name.clone(),
|
||||||
|
uploader_url: Some(yt_channel_url(&channel.id)),
|
||||||
|
thumbnail: v.thumbnail.last().map(|t| t.url.clone()),
|
||||||
|
duration_seconds: v.duration.unwrap_or(0) as i64,
|
||||||
|
view_count: v.view_count.unwrap_or(0) as i64,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let avatar = channel.avatar.last().map(|t| t.url.clone());
|
||||||
|
let banner = channel.banner.last().map(|t| t.url.clone());
|
||||||
|
|
||||||
|
Ok(ChannelInfo {
|
||||||
|
id: channel.id,
|
||||||
|
name: channel.name,
|
||||||
|
avatar,
|
||||||
|
banner,
|
||||||
|
subscriber_count: channel.subscriber_count.map(|n| n as i64).unwrap_or(-1),
|
||||||
|
description: channel.description,
|
||||||
|
videos,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
|
|
||||||
use std::sync::Once;
|
use std::sync::Once;
|
||||||
|
|
||||||
|
mod channel;
|
||||||
mod error;
|
mod error;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
mod search;
|
mod search;
|
||||||
|
|
@ -17,6 +18,7 @@ mod stream;
|
||||||
use runtime::block_on;
|
use runtime::block_on;
|
||||||
|
|
||||||
// Re-exports so UniFFI sees the types at the crate root for macro discovery.
|
// Re-exports so UniFFI sees the types at the crate root for macro discovery.
|
||||||
|
pub use channel::ChannelInfo;
|
||||||
pub use error::StrawcoreError;
|
pub use error::StrawcoreError;
|
||||||
pub use search::SearchItem;
|
pub use search::SearchItem;
|
||||||
pub use stream::{AudioStreamItem, StreamInfo, VideoStreamItem};
|
pub use stream::{AudioStreamItem, StreamInfo, VideoStreamItem};
|
||||||
|
|
|
||||||
|
|
@ -94,8 +94,8 @@ dependencies {
|
||||||
implementation(libs.coil.compose)
|
implementation(libs.coil.compose)
|
||||||
implementation(libs.coil.network.okhttp)
|
implementation(libs.coil.network.okhttp)
|
||||||
|
|
||||||
// NewPipeExtractor (JVM/Android-only) + its OkHttp dep
|
// Phase U-5: NewPipeExtractor (Java) torn out — strawcore handles
|
||||||
implementation(libs.newpipe.extractor)
|
// all YT extraction. OkHttp stays for RYD + SponsorBlock JSON clients.
|
||||||
implementation(libs.squareup.okhttp)
|
implementation(libs.squareup.okhttp)
|
||||||
|
|
||||||
// JSON for SponsorBlock + Return YouTube Dislike clients
|
// JSON for SponsorBlock + Return YouTube Dislike clients
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,9 @@
|
||||||
/*
|
/*
|
||||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
* SPDX-FileCopyrightText: 2026 Sulkta
|
||||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
*
|
||||||
|
* Phase U-5: NewPipeExtractor (Java) torn out. All YT extraction goes
|
||||||
|
* through strawcore (Rust + rustypipe + UniFFI) now.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.sulkta.straw
|
package com.sulkta.straw
|
||||||
|
|
@ -9,25 +12,16 @@ import android.app.Application
|
||||||
import com.sulkta.straw.data.History
|
import com.sulkta.straw.data.History
|
||||||
import com.sulkta.straw.data.Settings
|
import com.sulkta.straw.data.Settings
|
||||||
import com.sulkta.straw.data.Subscriptions
|
import com.sulkta.straw.data.Subscriptions
|
||||||
import com.sulkta.straw.extractor.NewPipeDownloader
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe
|
|
||||||
import org.schabi.newpipe.extractor.localization.ContentCountry
|
|
||||||
import org.schabi.newpipe.extractor.localization.Localization
|
|
||||||
|
|
||||||
class StrawApp : Application() {
|
class StrawApp : Application() {
|
||||||
override fun onCreate() {
|
override fun onCreate() {
|
||||||
super.onCreate()
|
super.onCreate()
|
||||||
NewPipe.init(
|
|
||||||
NewPipeDownloader.init(),
|
|
||||||
Localization("en", "US"),
|
|
||||||
ContentCountry("US"),
|
|
||||||
)
|
|
||||||
History.init(this)
|
History.init(this)
|
||||||
Settings.init(this)
|
Settings.init(this)
|
||||||
Subscriptions.init(this)
|
Subscriptions.init(this)
|
||||||
|
|
||||||
// Phase U-1: load the strawcore native library and route its logs
|
// Load strawcore native + route its logs into android logcat under
|
||||||
// into android logcat under the "strawcore" tag.
|
// the "strawcore" tag.
|
||||||
runCatching {
|
runCatching {
|
||||||
System.loadLibrary("strawcore")
|
System.loadLibrary("strawcore")
|
||||||
uniffi.strawcore.initLogging()
|
uniffi.strawcore.initLogging()
|
||||||
|
|
|
||||||
|
|
@ -1,96 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
|
||||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
*
|
|
||||||
* Minimal OkHttp-backed implementation of NewPipeExtractor's Downloader.
|
|
||||||
* No cookies, no recaptcha handling — anonymous browsing only. Modeled after
|
|
||||||
* NewPipe's DownloaderImpl but trimmed down for fork scope.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.sulkta.straw.extractor
|
|
||||||
|
|
||||||
import com.sulkta.straw.net.NEWPIPE_MAX_BYTES
|
|
||||||
import com.sulkta.straw.net.cappedString
|
|
||||||
import okhttp3.OkHttpClient
|
|
||||||
import okhttp3.RequestBody.Companion.toRequestBody
|
|
||||||
import org.schabi.newpipe.extractor.downloader.Downloader
|
|
||||||
import org.schabi.newpipe.extractor.downloader.Request
|
|
||||||
import org.schabi.newpipe.extractor.downloader.Response
|
|
||||||
import java.io.IOException
|
|
||||||
import java.util.concurrent.TimeUnit
|
|
||||||
|
|
||||||
class NewPipeDownloader private constructor(
|
|
||||||
private val client: OkHttpClient,
|
|
||||||
) : Downloader() {
|
|
||||||
|
|
||||||
override fun execute(request: Request): Response {
|
|
||||||
val httpMethod = request.httpMethod()
|
|
||||||
val url = request.url()
|
|
||||||
val headers = request.headers()
|
|
||||||
val data: ByteArray? = request.dataToSend()
|
|
||||||
|
|
||||||
val requestBody = data?.toRequestBody(null)
|
|
||||||
|
|
||||||
val okBuilder = okhttp3.Request.Builder()
|
|
||||||
.method(httpMethod, requestBody)
|
|
||||||
.url(url)
|
|
||||||
|
|
||||||
// AUD-HIGH: copy NPE headers BEFORE adding our explicit UA so the
|
|
||||||
// explicit UA wins; guard against header values containing \r/\n
|
|
||||||
// which OkHttp's addHeader rejects via IAE (turning a poisoned
|
|
||||||
// response into an app crash).
|
|
||||||
headers.forEach { (name, values) ->
|
|
||||||
if (name.equals("User-Agent", ignoreCase = true)) return@forEach
|
|
||||||
okBuilder.removeHeader(name)
|
|
||||||
values.forEach { value ->
|
|
||||||
runCatching { okBuilder.addHeader(name, value) }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
okBuilder.removeHeader("User-Agent")
|
|
||||||
okBuilder.addHeader("User-Agent", USER_AGENT)
|
|
||||||
|
|
||||||
val okResponse = client.newCall(okBuilder.build()).execute()
|
|
||||||
val body = okResponse.body
|
|
||||||
// AUD-HIGH: bounded read to defend against OOM via gigabyte response.
|
|
||||||
val bodyString = body?.cappedString(NEWPIPE_MAX_BYTES) ?: ""
|
|
||||||
val responseHeaders = okResponse.headers.toMultimap()
|
|
||||||
val latestUrl = okResponse.request.url.toString()
|
|
||||||
if (okResponse.code == 429) {
|
|
||||||
okResponse.close()
|
|
||||||
throw IOException("HTTP 429 — rate limited")
|
|
||||||
}
|
|
||||||
okResponse.close()
|
|
||||||
|
|
||||||
return Response(
|
|
||||||
okResponse.code,
|
|
||||||
okResponse.message,
|
|
||||||
responseHeaders,
|
|
||||||
bodyString,
|
|
||||||
latestUrl,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
companion object {
|
|
||||||
const val USER_AGENT =
|
|
||||||
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) " +
|
|
||||||
"Chrome/120.0.0.0 Mobile Safari/537.36"
|
|
||||||
|
|
||||||
@Volatile private var instance: NewPipeDownloader? = null
|
|
||||||
|
|
||||||
fun init(builder: OkHttpClient.Builder? = null): NewPipeDownloader {
|
|
||||||
val client = (builder ?: OkHttpClient.Builder())
|
|
||||||
.connectTimeout(30, TimeUnit.SECONDS)
|
|
||||||
.readTimeout(30, TimeUnit.SECONDS)
|
|
||||||
.writeTimeout(30, TimeUnit.SECONDS)
|
|
||||||
.build()
|
|
||||||
val d = NewPipeDownloader(client)
|
|
||||||
instance = d
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
|
|
||||||
fun get(): NewPipeDownloader = instance
|
|
||||||
?: error("NewPipeDownloader not initialized — call init() first")
|
|
||||||
|
|
||||||
fun client(): OkHttpClient = get().client
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,6 +1,10 @@
|
||||||
/*
|
/*
|
||||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
* SPDX-FileCopyrightText: 2026 Sulkta
|
||||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
*
|
||||||
|
* Phase U-4: ChannelInfo + Videos tab moved to strawcore (rustypipe).
|
||||||
|
* The two separate ChannelInfo.getInfo + ChannelTabInfo.getInfo calls
|
||||||
|
* collapse into one Rust round-trip.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.sulkta.straw.feature.channel
|
package com.sulkta.straw.feature.channel
|
||||||
|
|
@ -8,19 +12,10 @@ package com.sulkta.straw.feature.channel
|
||||||
import androidx.lifecycle.ViewModel
|
import androidx.lifecycle.ViewModel
|
||||||
import androidx.lifecycle.viewModelScope
|
import androidx.lifecycle.viewModelScope
|
||||||
import com.sulkta.straw.feature.search.StreamItem
|
import com.sulkta.straw.feature.search.StreamItem
|
||||||
import com.sulkta.straw.util.bestThumbnail
|
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import org.schabi.newpipe.extractor.NewPipe
|
|
||||||
import org.schabi.newpipe.extractor.ServiceList
|
|
||||||
import kotlinx.coroutines.flow.MutableStateFlow
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
import kotlinx.coroutines.flow.StateFlow
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
import kotlinx.coroutines.flow.asStateFlow
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.withContext
|
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfo
|
|
||||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
|
|
||||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem
|
|
||||||
|
|
||||||
data class ChannelUiState(
|
data class ChannelUiState(
|
||||||
val loading: Boolean = true,
|
val loading: Boolean = true,
|
||||||
|
|
@ -40,43 +35,24 @@ class ChannelViewModel : ViewModel() {
|
||||||
_ui.value = ChannelUiState(loading = true)
|
_ui.value = ChannelUiState(loading = true)
|
||||||
viewModelScope.launch {
|
viewModelScope.launch {
|
||||||
try {
|
try {
|
||||||
val service = NewPipe.getService(ServiceList.YouTube.serviceId)
|
val ch = uniffi.strawcore.channelInfo(channelUrl)
|
||||||
val info = withContext(Dispatchers.IO) {
|
val videos = ch.videos.map { v ->
|
||||||
ChannelInfo.getInfo(service, channelUrl)
|
StreamItem(
|
||||||
|
url = v.url,
|
||||||
|
title = v.title.ifBlank { "(no title)" },
|
||||||
|
uploader = v.uploader,
|
||||||
|
uploaderUrl = v.uploaderUrl,
|
||||||
|
thumbnail = v.thumbnail,
|
||||||
|
durationSeconds = v.durationSeconds,
|
||||||
|
viewCount = v.viewCount,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
// AUD-HIGH: pick the Videos tab specifically rather than
|
|
||||||
// info.tabs.firstOrNull() which is YouTube's "Home" (a
|
|
||||||
// curated mix that mostly drops via filterIsInstance).
|
|
||||||
val videosTab = info.tabs.firstOrNull {
|
|
||||||
it.contentFilters.contains(ChannelTabs.VIDEOS)
|
|
||||||
} ?: info.tabs.firstOrNull()
|
|
||||||
val videos: List<StreamItem> = if (videosTab != null) {
|
|
||||||
withContext(Dispatchers.IO) {
|
|
||||||
runCatching {
|
|
||||||
ChannelTabInfo.getInfo(service, videosTab)
|
|
||||||
.relatedItems
|
|
||||||
.filterIsInstance<StreamInfoItem>()
|
|
||||||
.map {
|
|
||||||
StreamItem(
|
|
||||||
url = it.url,
|
|
||||||
title = it.name ?: "(no title)",
|
|
||||||
uploader = it.uploaderName ?: info.name ?: "",
|
|
||||||
uploaderUrl = it.uploaderUrl ?: channelUrl,
|
|
||||||
thumbnail = bestThumbnail(it.thumbnails),
|
|
||||||
durationSeconds = it.duration,
|
|
||||||
viewCount = it.viewCount,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}.getOrDefault(emptyList())
|
|
||||||
}
|
|
||||||
} else emptyList()
|
|
||||||
|
|
||||||
_ui.value = ChannelUiState(
|
_ui.value = ChannelUiState(
|
||||||
loading = false,
|
loading = false,
|
||||||
name = info.name ?: "",
|
name = ch.name,
|
||||||
subscriberCount = info.subscriberCount,
|
subscriberCount = ch.subscriberCount,
|
||||||
banner = bestThumbnail(info.banners),
|
banner = ch.banner,
|
||||||
avatar = bestThumbnail(info.avatars),
|
avatar = ch.avatar,
|
||||||
videos = videos,
|
videos = videos,
|
||||||
)
|
)
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,14 @@
|
||||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
* SPDX-FileCopyrightText: 2026 Sulkta
|
||||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
*
|
*
|
||||||
* Phase Q: aggregate latest videos across all subscribed channels into a
|
* Phase Q + U-4: aggregate latest videos across all subscribed channels.
|
||||||
* single feed. Fans out per-channel ChannelInfo + ChannelTabs.VIDEOS
|
* Per-channel fetches now go through strawcore (rustypipe + UniFFI). The
|
||||||
* fetches in parallel, merges by view count desc, caps at 200 items.
|
* Kotlin side still owns the structured concurrency:
|
||||||
*
|
* - cancels prior in-flight refresh
|
||||||
* Audit fixes (2026-05-24 pass #2):
|
* - Semaphore caps parallel fetches (rustypipe internally has its own
|
||||||
* HIGH-6: cancel any prior in-flight refresh when a new one starts, cap
|
* HTTP pool but we still want to avoid N=100 concurrent extractor
|
||||||
* concurrency with a Semaphore, time-bound each per-channel fetch so
|
* contexts when N=100 channels)
|
||||||
* one hung channel can't stall the whole feed.
|
* - per-channel 15s timeout
|
||||||
* MED-7: use `update { }` for atomic UI-state writes (matches the
|
|
||||||
* convention applied to the data stores in audit pass #1).
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package com.sulkta.straw.feature.feed
|
package com.sulkta.straw.feature.feed
|
||||||
|
|
@ -20,9 +18,7 @@ import androidx.lifecycle.ViewModel
|
||||||
import androidx.lifecycle.viewModelScope
|
import androidx.lifecycle.viewModelScope
|
||||||
import com.sulkta.straw.data.Subscriptions
|
import com.sulkta.straw.data.Subscriptions
|
||||||
import com.sulkta.straw.feature.search.StreamItem
|
import com.sulkta.straw.feature.search.StreamItem
|
||||||
import com.sulkta.straw.util.bestThumbnail
|
|
||||||
import com.sulkta.straw.util.strawLogW
|
import com.sulkta.straw.util.strawLogW
|
||||||
import kotlinx.coroutines.Dispatchers
|
|
||||||
import kotlinx.coroutines.Job
|
import kotlinx.coroutines.Job
|
||||||
import kotlinx.coroutines.async
|
import kotlinx.coroutines.async
|
||||||
import kotlinx.coroutines.awaitAll
|
import kotlinx.coroutines.awaitAll
|
||||||
|
|
@ -34,14 +30,7 @@ import kotlinx.coroutines.flow.update
|
||||||
import kotlinx.coroutines.launch
|
import kotlinx.coroutines.launch
|
||||||
import kotlinx.coroutines.sync.Semaphore
|
import kotlinx.coroutines.sync.Semaphore
|
||||||
import kotlinx.coroutines.sync.withPermit
|
import kotlinx.coroutines.sync.withPermit
|
||||||
import kotlinx.coroutines.withContext
|
|
||||||
import kotlinx.coroutines.withTimeoutOrNull
|
import kotlinx.coroutines.withTimeoutOrNull
|
||||||
import org.schabi.newpipe.extractor.NewPipe
|
|
||||||
import org.schabi.newpipe.extractor.ServiceList
|
|
||||||
import org.schabi.newpipe.extractor.channel.ChannelInfo
|
|
||||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabInfo
|
|
||||||
import org.schabi.newpipe.extractor.channel.tabs.ChannelTabs
|
|
||||||
import org.schabi.newpipe.extractor.stream.StreamInfoItem
|
|
||||||
|
|
||||||
data class SubscriptionFeedUiState(
|
data class SubscriptionFeedUiState(
|
||||||
val loading: Boolean = false,
|
val loading: Boolean = false,
|
||||||
|
|
@ -54,16 +43,11 @@ class SubscriptionFeedViewModel : ViewModel() {
|
||||||
private val _ui = MutableStateFlow(SubscriptionFeedUiState())
|
private val _ui = MutableStateFlow(SubscriptionFeedUiState())
|
||||||
val ui: StateFlow<SubscriptionFeedUiState> = _ui.asStateFlow()
|
val ui: StateFlow<SubscriptionFeedUiState> = _ui.asStateFlow()
|
||||||
|
|
||||||
/** Cache feed for 10 min to avoid hammering YT on tab re-entry. */
|
|
||||||
private val cacheTtlMs = 10L * 60 * 1000
|
private val cacheTtlMs = 10L * 60 * 1000
|
||||||
|
|
||||||
/** Per-channel fetch timeout — slowest channel can't stall the whole batch. */
|
|
||||||
private val perChannelTimeoutMs = 15_000L
|
private val perChannelTimeoutMs = 15_000L
|
||||||
|
|
||||||
/** Cap parallel network fetches even with 100+ subs. */
|
|
||||||
private val parallelism = 8
|
private val parallelism = 8
|
||||||
|
private val perChannelMax = 5
|
||||||
|
|
||||||
/** Live refresh job, so spam-tapping Refresh doesn't fan out racing fetches. */
|
|
||||||
private var inFlight: Job? = null
|
private var inFlight: Job? = null
|
||||||
|
|
||||||
fun refreshIfStale() {
|
fun refreshIfStale() {
|
||||||
|
|
@ -82,55 +66,40 @@ class SubscriptionFeedViewModel : ViewModel() {
|
||||||
_ui.update { it.copy(loading = true, error = null) }
|
_ui.update { it.copy(loading = true, error = null) }
|
||||||
inFlight = viewModelScope.launch {
|
inFlight = viewModelScope.launch {
|
||||||
try {
|
try {
|
||||||
val items = withContext(Dispatchers.IO) {
|
val gate = Semaphore(parallelism)
|
||||||
val service = NewPipe.getService(ServiceList.YouTube.serviceId)
|
val items = coroutineScope {
|
||||||
val perChannelMax = 5
|
val deferreds = channels.map { ch ->
|
||||||
val gate = Semaphore(parallelism)
|
async {
|
||||||
coroutineScope {
|
gate.withPermit {
|
||||||
val deferreds = channels.map { ch ->
|
withTimeoutOrNull(perChannelTimeoutMs) {
|
||||||
async {
|
runCatching {
|
||||||
gate.withPermit {
|
uniffi.strawcore.channelInfo(ch.url).videos.take(perChannelMax).map { v ->
|
||||||
withTimeoutOrNull(perChannelTimeoutMs) {
|
StreamItem(
|
||||||
runCatching {
|
url = v.url,
|
||||||
val info = ChannelInfo.getInfo(service, ch.url)
|
title = v.title.ifBlank { "(no title)" },
|
||||||
val tab = info.tabs.firstOrNull {
|
uploader = v.uploader.ifBlank { ch.name },
|
||||||
it.contentFilters.contains(ChannelTabs.VIDEOS)
|
uploaderUrl = v.uploaderUrl ?: ch.url,
|
||||||
} ?: info.tabs.firstOrNull()
|
thumbnail = v.thumbnail,
|
||||||
?: return@runCatching emptyList<StreamItem>()
|
durationSeconds = v.durationSeconds,
|
||||||
ChannelTabInfo.getInfo(service, tab)
|
viewCount = v.viewCount,
|
||||||
.relatedItems
|
)
|
||||||
.filterIsInstance<StreamInfoItem>()
|
}
|
||||||
.take(perChannelMax)
|
}.onFailure {
|
||||||
.map { si ->
|
strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" }
|
||||||
StreamItem(
|
}.getOrDefault(emptyList())
|
||||||
url = si.url,
|
} ?: run {
|
||||||
title = si.name ?: "(no title)",
|
strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" }
|
||||||
uploader = si.uploaderName ?: ch.name,
|
emptyList()
|
||||||
uploaderUrl = si.uploaderUrl ?: ch.url,
|
|
||||||
thumbnail = bestThumbnail(si.thumbnails),
|
|
||||||
durationSeconds = si.duration,
|
|
||||||
viewCount = si.viewCount,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}.onFailure {
|
|
||||||
strawLogW("StrawFeed") { "channel fetch failed for ${ch.url}: ${it.message}" }
|
|
||||||
}.getOrDefault(emptyList())
|
|
||||||
} ?: run {
|
|
||||||
strawLogW("StrawFeed") { "channel fetch timed out: ${ch.url}" }
|
|
||||||
emptyList()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
deferreds.awaitAll()
|
|
||||||
}
|
}
|
||||||
.flatten()
|
deferreds.awaitAll()
|
||||||
// No reliable upload-timestamp from extractor's StreamInfoItem
|
|
||||||
// in all cases — sort by view count desc as a soft proxy for
|
|
||||||
// recency-popularity within the recent window.
|
|
||||||
.sortedByDescending { it.viewCount }
|
|
||||||
.take(200)
|
|
||||||
}
|
}
|
||||||
|
.flatten()
|
||||||
|
.sortedByDescending { it.viewCount }
|
||||||
|
.take(200)
|
||||||
|
|
||||||
_ui.update {
|
_ui.update {
|
||||||
SubscriptionFeedUiState(
|
SubscriptionFeedUiState(
|
||||||
loading = false,
|
loading = false,
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,7 @@ import androidx.media3.exoplayer.source.DefaultMediaSourceFactory
|
||||||
import androidx.media3.session.MediaSession
|
import androidx.media3.session.MediaSession
|
||||||
import androidx.media3.session.MediaSessionService
|
import androidx.media3.session.MediaSessionService
|
||||||
import com.sulkta.straw.StrawActivity
|
import com.sulkta.straw.StrawActivity
|
||||||
import com.sulkta.straw.extractor.NewPipeDownloader
|
import com.sulkta.straw.net.STRAW_USER_AGENT
|
||||||
|
|
||||||
@UnstableApi
|
@UnstableApi
|
||||||
class PlaybackService : MediaSessionService() {
|
class PlaybackService : MediaSessionService() {
|
||||||
|
|
@ -63,7 +63,7 @@ class PlaybackService : MediaSessionService() {
|
||||||
ensureChannel()
|
ensureChannel()
|
||||||
|
|
||||||
val httpFactory = DefaultHttpDataSource.Factory()
|
val httpFactory = DefaultHttpDataSource.Factory()
|
||||||
.setUserAgent(NewPipeDownloader.USER_AGENT)
|
.setUserAgent(STRAW_USER_AGENT)
|
||||||
.setAllowCrossProtocolRedirects(true)
|
.setAllowCrossProtocolRedirects(true)
|
||||||
val mediaSourceFactory = DefaultMediaSourceFactory(this)
|
val mediaSourceFactory = DefaultMediaSourceFactory(this)
|
||||||
.setDataSourceFactory(httpFactory)
|
.setDataSourceFactory(httpFactory)
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ import androidx.media3.exoplayer.hls.HlsMediaSource
|
||||||
import androidx.media3.exoplayer.source.MergingMediaSource
|
import androidx.media3.exoplayer.source.MergingMediaSource
|
||||||
import androidx.media3.exoplayer.source.ProgressiveMediaSource
|
import androidx.media3.exoplayer.source.ProgressiveMediaSource
|
||||||
import androidx.media3.ui.PlayerView
|
import androidx.media3.ui.PlayerView
|
||||||
import com.sulkta.straw.extractor.NewPipeDownloader
|
import com.sulkta.straw.net.STRAW_USER_AGENT
|
||||||
import com.sulkta.straw.net.SbSegment
|
import com.sulkta.straw.net.SbSegment
|
||||||
import com.sulkta.straw.util.strawLogI
|
import com.sulkta.straw.util.strawLogI
|
||||||
import kotlinx.coroutines.delay
|
import kotlinx.coroutines.delay
|
||||||
|
|
@ -169,7 +169,7 @@ fun PlayerScreen(
|
||||||
LaunchedEffect(resolved) {
|
LaunchedEffect(resolved) {
|
||||||
val r = resolved ?: return@LaunchedEffect
|
val r = resolved ?: return@LaunchedEffect
|
||||||
val dataSourceFactory = DefaultHttpDataSource.Factory()
|
val dataSourceFactory = DefaultHttpDataSource.Factory()
|
||||||
.setUserAgent(NewPipeDownloader.USER_AGENT)
|
.setUserAgent(STRAW_USER_AGENT)
|
||||||
.setAllowCrossProtocolRedirects(true)
|
.setAllowCrossProtocolRedirects(true)
|
||||||
|
|
||||||
val source = when {
|
val source = when {
|
||||||
|
|
|
||||||
|
|
@ -13,9 +13,33 @@
|
||||||
|
|
||||||
package com.sulkta.straw.net
|
package com.sulkta.straw.net
|
||||||
|
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
import okhttp3.ResponseBody
|
import okhttp3.ResponseBody
|
||||||
import okio.Buffer
|
import okio.Buffer
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Phase U-5: USER_AGENT + shared OkHttpClient that previously lived on
|
||||||
|
* NewPipeDownloader. After ripping NewPipeExtractor, the RYD + SponsorBlock
|
||||||
|
* + ExoPlayer HTTP factories still need both. One shared client is fine.
|
||||||
|
*/
|
||||||
|
const val STRAW_USER_AGENT: String =
|
||||||
|
"Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36 Straw/1.0"
|
||||||
|
|
||||||
|
@Volatile
|
||||||
|
private var sharedClient: OkHttpClient? = null
|
||||||
|
|
||||||
|
fun strawHttpClient(): OkHttpClient =
|
||||||
|
sharedClient ?: synchronized(STRAW_USER_AGENT) {
|
||||||
|
sharedClient ?: OkHttpClient.Builder()
|
||||||
|
.connectTimeout(15, TimeUnit.SECONDS)
|
||||||
|
.readTimeout(30, TimeUnit.SECONDS)
|
||||||
|
.followRedirects(true)
|
||||||
|
.followSslRedirects(true)
|
||||||
|
.build()
|
||||||
|
.also { sharedClient = it }
|
||||||
|
}
|
||||||
|
|
||||||
fun ResponseBody.cappedString(maxBytes: Long): String {
|
fun ResponseBody.cappedString(maxBytes: Long): String {
|
||||||
val cl = contentLength()
|
val cl = contentLength()
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@
|
||||||
|
|
||||||
package com.sulkta.straw.net
|
package com.sulkta.straw.net
|
||||||
|
|
||||||
import com.sulkta.straw.extractor.NewPipeDownloader
|
|
||||||
import com.sulkta.straw.util.strawLogD
|
import com.sulkta.straw.util.strawLogD
|
||||||
import com.sulkta.straw.util.strawLogW
|
import com.sulkta.straw.util.strawLogW
|
||||||
import kotlinx.serialization.Serializable
|
import kotlinx.serialization.Serializable
|
||||||
|
|
@ -34,11 +33,11 @@ object RydClient {
|
||||||
strawLogD(TAG) { "fetch start: $videoId → $url" }
|
strawLogD(TAG) { "fetch start: $videoId → $url" }
|
||||||
val req = Request.Builder()
|
val req = Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.header("User-Agent", NewPipeDownloader.USER_AGENT)
|
.header("User-Agent", STRAW_USER_AGENT)
|
||||||
.header("Accept", "application/json")
|
.header("Accept", "application/json")
|
||||||
.build()
|
.build()
|
||||||
return runCatching {
|
return runCatching {
|
||||||
NewPipeDownloader.client().newCall(req).execute().use { r ->
|
strawHttpClient().newCall(req).execute().use { r ->
|
||||||
val code = r.code
|
val code = r.code
|
||||||
// AUD-HIGH: bounded body read to defend against OOM.
|
// AUD-HIGH: bounded body read to defend against OOM.
|
||||||
val bodyStr = r.body?.cappedString(RYD_MAX_BYTES) ?: ""
|
val bodyStr = r.body?.cappedString(RYD_MAX_BYTES) ?: ""
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@
|
||||||
|
|
||||||
package com.sulkta.straw.net
|
package com.sulkta.straw.net
|
||||||
|
|
||||||
import com.sulkta.straw.extractor.NewPipeDownloader
|
|
||||||
import com.sulkta.straw.util.strawLogD
|
import com.sulkta.straw.util.strawLogD
|
||||||
import com.sulkta.straw.util.strawLogW
|
import com.sulkta.straw.util.strawLogW
|
||||||
import kotlinx.serialization.Serializable
|
import kotlinx.serialization.Serializable
|
||||||
|
|
@ -47,11 +46,11 @@ object SponsorBlockClient {
|
||||||
strawLogD(TAG) { "fetch: videoId=$videoId prefix=$prefix url=$urlStr" }
|
strawLogD(TAG) { "fetch: videoId=$videoId prefix=$prefix url=$urlStr" }
|
||||||
val req = Request.Builder()
|
val req = Request.Builder()
|
||||||
.url(urlStr)
|
.url(urlStr)
|
||||||
.header("User-Agent", NewPipeDownloader.USER_AGENT)
|
.header("User-Agent", STRAW_USER_AGENT)
|
||||||
.header("Accept", "application/json")
|
.header("Accept", "application/json")
|
||||||
.build()
|
.build()
|
||||||
return runCatching {
|
return runCatching {
|
||||||
NewPipeDownloader.client().newCall(req).execute().use { r ->
|
strawHttpClient().newCall(req).execute().use { r ->
|
||||||
val code = r.code
|
val code = r.code
|
||||||
// AUD-HIGH: bounded body read.
|
// AUD-HIGH: bounded body read.
|
||||||
val bodyStr = r.body?.cappedString(SB_MAX_BYTES) ?: ""
|
val bodyStr = r.body?.cappedString(SB_MAX_BYTES) ?: ""
|
||||||
|
|
|
||||||
|
|
@ -1,24 +0,0 @@
|
||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
|
||||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
|
||||||
*
|
|
||||||
* NewPipeExtractor returns thumbnails as a List<Image> with width/height
|
|
||||||
* fields. Calling .firstOrNull() picks the smallest (the list is sorted
|
|
||||||
* ascending) — which gave us pixelated thumbnails. This helper picks the
|
|
||||||
* largest by pixel area instead.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package com.sulkta.straw.util
|
|
||||||
|
|
||||||
import org.schabi.newpipe.extractor.Image
|
|
||||||
|
|
||||||
fun bestThumbnail(images: List<Image>?): String? {
|
|
||||||
if (images.isNullOrEmpty()) return null
|
|
||||||
return images
|
|
||||||
.maxByOrNull {
|
|
||||||
val w = it.width.takeIf { v -> v > 0 } ?: 0
|
|
||||||
val h = it.height.takeIf { v -> v > 0 } ?: 0
|
|
||||||
w.toLong() * h.toLong()
|
|
||||||
}
|
|
||||||
?.url
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue