Compare commits

..

1 commit

10 changed files with 33 additions and 264 deletions

1
rust/Cargo.lock generated
View file

@ -1571,7 +1571,6 @@ dependencies = [
name = "strawcore-core" name = "strawcore-core"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"log",
"once_cell", "once_cell",
"parking_lot", "parking_lot",
"regex", "regex",

View file

@ -33,11 +33,7 @@ pub fn init_logging() {
ONCE.call_once(|| { ONCE.call_once(|| {
android_logger::init_once( android_logger::init_once(
android_logger::Config::default() android_logger::Config::default()
// Debug (not Info) so the extractor's DEBUG tier — per-request .with_max_level(log::LevelFilter::Info)
// HTTP breadcrumbs, cache/lexer-fallback internals — is live in
// this dogfood/debug build. WARN/INFO were already emitted; this
// unlocks the chatty diagnostics behind them.
.with_max_level(log::LevelFilter::Debug)
.with_tag("strawcore"), .with_tag("strawcore"),
); );
log::info!("strawcore initialized"); log::info!("strawcore initialized");

View file

@ -122,10 +122,7 @@ pub(crate) async fn run_extract<T, E, F>(what: &'static str, f: F) -> Result<T,
where where
F: FnOnce() -> Result<T, E> + Send + 'static, F: FnOnce() -> Result<T, E> + Send + 'static,
T: Send + 'static, T: Send + 'static,
// `Display` so the single catch-all below can log the error. Every caller E: Send + 'static,
// passes a core `ExtractionError`, whose Display is already URL/token
// scrubbed at the source (exceptions.rs choke points), so this is safe.
E: std::fmt::Display + Send + 'static,
StrawcoreError: From<E>, StrawcoreError: From<E>,
{ {
match tokio::time::timeout(EXTRACT_TIMEOUT, tokio::task::spawn_blocking(f)).await { match tokio::time::timeout(EXTRACT_TIMEOUT, tokio::task::spawn_blocking(f)).await {
@ -139,14 +136,7 @@ where
msg: format!("join: {join}"), msg: format!("join: {join}"),
}), }),
// Blocking task returned; propagate its own error via the existing // Blocking task returned; propagate its own error via the existing
// `From<ExtractionError>` mapping. This is the single catch-all for // `From<ExtractionError>` mapping.
// EVERY extraction failure system-wide (incl. the bot-wall) — one WARN Ok(Ok(inner)) => inner.map_err(StrawcoreError::from),
// here means no failure path is silent. Display strings are scrubbed
// at the source, so this never leaks a stream URL or token.
Ok(Ok(Ok(v))) => Ok(v),
Ok(Ok(Err(err))) => {
log::warn!("strawcore::{what} failed: {err}");
Err(StrawcoreError::from(err))
}
} }
} }

View file

@ -203,7 +203,7 @@ dependencies {
// strawcore — Rust YouTube extractor via UniFFI/JNA. Built by the // strawcore — Rust YouTube extractor via UniFFI/JNA. Built by the
// cargoBuild + uniffiBindgen tasks below; phase U-2+ exposes search / // cargoBuild + uniffiBindgen tasks below; phase U-2+ exposes search /
// streamInfo / channelInfo to replace NewPipeExtractor. // streamInfo / channelInfo to replace NewPipeExtractor.
implementation("net.java.dev.jna:jna:5.14.0@aar") implementation("net.java.dev.jna:jna:5.19.1@aar")
} }
// ============================================================================= // =============================================================================

View file

@ -26,7 +26,6 @@ import com.sulkta.straw.net.SponsorBlockClient
import com.sulkta.straw.feature.search.StreamItem import com.sulkta.straw.feature.search.StreamItem
import com.sulkta.straw.util.isAllowedYtUrl import com.sulkta.straw.util.isAllowedYtUrl
import com.sulkta.straw.util.runCatchingCancellable import com.sulkta.straw.util.runCatchingCancellable
import com.sulkta.straw.util.strawLogI
import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job import kotlinx.coroutines.Job
@ -331,18 +330,12 @@ class VideoDetailViewModel : ViewModel() {
} catch (t: Throwable) { } catch (t: Throwable) {
if (t is CancellationException) throw t if (t is CancellationException) throw t
if (_ui.value.loadedUrl != streamUrl) return@launch if (_ui.value.loadedUrl != streamUrl) return@launch
// FULL scrub — this reason is rendered to the UI below (and so
// is screenshot-reachable). The always-log line seeds the
// dogfood ring with the user-visible extraction failure; the
// ring's own (lighter) scrub applies at send time.
val reason = com.sulkta.straw.util.LogDump.scrubLine(
t.message ?: t.javaClass.simpleName,
)
strawLogI("StrawDetail", "extraction failed: $reason")
_ui.update { _ui.update {
VideoDetailUiState( VideoDetailUiState(
loading = false, loading = false,
error = reason, error = com.sulkta.straw.util.LogDump.scrubLine(
t.message ?: t.javaClass.simpleName,
),
loadedUrl = streamUrl, loadedUrl = streamUrl,
) )
} }

View file

@ -13,11 +13,8 @@
* * manual the Settings Diagnostics "Send logs to Kayos" button. * * manual the Settings Diagnostics "Send logs to Kayos" button.
* *
* PRIVACY: every line including the crash stacktrace goes through * PRIVACY: every line including the crash stacktrace goes through
* LogDump.scrubLineDogfood before leaving the device (the lighter * LogDump.scrubLine before leaving the device; the server re-scrubs,
* profile for our own logs.sulkta.com: it still redacts every * but the phone is the primary gate. device_id is a per-install random
* credential/URL/token, but keeps bare video/channel/playlist ids as
* outage-triage signal); the server re-scrubs, but the phone is the
* primary gate. device_id is a per-install random
* UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole * UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole
* feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was * feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was
* injected (see strawApp/build.gradle.kts) for everyone else * injected (see strawApp/build.gradle.kts) for everyone else
@ -175,11 +172,8 @@ object LogShipper {
throwable.printStackTrace(PrintWriter(it)) throwable.printStackTrace(PrintWriter(it))
}.toString() }.toString()
.split('\n') .split('\n')
// Dogfood profile: this whole shipper targets our own
// logs.sulkta.com, so keep diagnostic ids while still
// scrubbing every credential/URL/token the trace may carry.
.take(400) .take(400)
.map(LogDump::scrubLineDogfood) .map(LogDump::scrubLine)
val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES) val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES)
postLines( postLines(
context = context, context = context,

View file

@ -105,7 +105,6 @@ import com.sulkta.straw.data.Settings
import com.sulkta.straw.feature.detail.VideoDetailBody import com.sulkta.straw.feature.detail.VideoDetailBody
import com.sulkta.straw.feature.detail.VideoDetailViewModel import com.sulkta.straw.feature.detail.VideoDetailViewModel
import com.sulkta.straw.util.LogDump import com.sulkta.straw.util.LogDump
import com.sulkta.straw.util.strawLogI
import kotlinx.coroutines.launch import kotlinx.coroutines.launch
// Snappy, no overshoot. A spring adapts its speed to the remaining // Snappy, no overshoot. A spring adapts its speed to the remaining
@ -483,12 +482,7 @@ private fun InlinePlayerSurface(
val listener = object : Player.Listener { val listener = object : Player.Listener {
override fun onPlayerError(error: androidx.media3.common.PlaybackException) { override fun onPlayerError(error: androidx.media3.common.PlaybackException) {
val raw = error.message ?: "(no message)" val raw = error.message ?: "(no message)"
// FULL scrub — this is rendered in the on-screen error banner. playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
playbackError = scrubbed
// Always-log so the dogfood ring keeps the user-visible
// playback failure (the ring's own scrub applies at send time).
strawLogI("StrawPlayer", "playback error: $scrubbed")
NowPlaying.clear() NowPlaying.clear()
} }
} }

View file

@ -132,12 +132,7 @@ fun PlayerScreen(
// string — visible in the on-screen error banner and // string — visible in the on-screen error banner and
// a screenshot away from being shared. // a screenshot away from being shared.
val raw = error.message ?: "(no message)" val raw = error.message ?: "(no message)"
// FULL scrub — this is rendered in the on-screen error banner. playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
playbackError = scrubbed
// Always-log so the dogfood ring keeps the user-visible
// playback failure (the ring's own scrub applies at send time).
strawLogI("StrawPlayer", "playback error: $scrubbed")
// Also clear NowPlaying so the minibar doesn't keep // Also clear NowPlaying so the minibar doesn't keep
// claiming a dead session is loaded. // claiming a dead session is loaded.
NowPlaying.clear() NowPlaying.clear()

View file

@ -122,9 +122,7 @@ object LogDump {
} }
// Scrub only the retained window — cheaper than scrubbing lines // Scrub only the retained window — cheaper than scrubbing lines
// that get dropped, and nothing raw ever leaves this function. // that get dropped, and nothing raw ever leaves this function.
// This is the dogfood ingest path (LogShipper → logs.sulkta.com, our return ring.map(::scrubLine)
// own infra), so use the lighter profile that keeps diagnostic ids.
return ring.map(::scrubLineDogfood)
} }
/** /**
@ -134,13 +132,10 @@ object LogDump {
* approach catches every documented leak vector at zero * approach catches every documented leak vector at zero
* allocation cost. * allocation cost.
* *
* BIAS: over-redaction. This FULL profile backs the share-sheet * BIAS: over-redaction. These lines leave the device (share sheet
* export and every on-screen error string, which can be handed to a * + LogShipper logs.sulkta.com), so a false positive costs a
* chooser app or screenshotted, so a false positive costs a little * little debug signal while a false negative leaks what someone
* debug signal while a false negative leaks what someone watched. * watched. Anything URL-, token-, id-, or address-shaped goes.
* Anything URL-, token-, id-, or address-shaped goes. (The dogfood
* ingest path to our own logs.sulkta.com uses the lighter
* [scrubLineDogfood] instead it keeps bare identifiers.)
* *
* Public so error-handler call sites (PlayerScreen / VideoDetail * Public so error-handler call sites (PlayerScreen / VideoDetail
* `playbackError`) can scrub Media3's `PlaybackException.message` * `playbackError`) can scrub Media3's `PlaybackException.message`
@ -148,30 +143,7 @@ object LogDump {
* request URI for HttpDataSource exceptions, which would otherwise * request URI for HttpDataSource exceptions, which would otherwise
* be a leak via screenshot. * be a leak via screenshot.
*/ */
fun scrubLine(line: String): String = scrub(line, keepIdentifiers = false) fun scrubLine(line: String): String {
/**
* DOGFOOD profile for the logs.sulkta.com ingest path ONLY (our own
* infra, driven by [LogShipper]). Scrubs every real credential / PII
* vector exactly as [scrubLine] does (googlevideo URLs, bearer tokens,
* cred headers/params, signed params, URL tails, emails, high-entropy
* tokens, IPs), but KEEPS the bare YouTube identifiers 11-char video
* ids, channel/playlist ids, schemeless youtu.be links because those
* are the diagnostic signal we triage an outage on, and they only ever
* reach a server we run. NEVER use this for the share-sheet export or a
* string rendered to the UI; those keep the FULL [scrubLine] profile.
*/
fun scrubLineDogfood(line: String): String = scrub(line, keepIdentifiers = true)
/**
* Shared scrub core. [keepIdentifiers] = true selects the lighter
* DOGFOOD profile: it SKIPS the four identifier replacements (schemeless
* youtu.be, channel id, playlist id, bare video id) so those survive,
* while every credential / high-entropy / IP vector is still redacted in
* BOTH profiles. Step ORDER is unchanged from the original full pass, so
* `keepIdentifiers = false` is byte-for-byte the old behavior.
*/
private fun scrub(line: String, keepIdentifiers: Boolean): String {
var s = line var s = line
// Pre-signed googlevideo URLs: keep the host label visible, drop // Pre-signed googlevideo URLs: keep the host label visible, drop
// host prefix + path + query (session-bound streaming creds). // host prefix + path + query (session-bound streaming creds).
@ -190,15 +162,10 @@ object LogDump {
// /vi/<id>/ thumbnails, search terms, etc. live. // /vi/<id>/ thumbnails, search terms, etc. live.
s = URL_TAIL_RE.replace(s, "$1/<scrubbed>") s = URL_TAIL_RE.replace(s, "$1/<scrubbed>")
// Schemeless YT links pasted into log messages ("youtu.be/xyz"). // Schemeless YT links pasted into log messages ("youtu.be/xyz").
// KEPT on the dogfood profile — the id is diagnostic signal.
if (!keepIdentifiers) {
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>") s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
}
// Emails. // Emails.
s = EMAIL_RE.replace(s, "<email>") s = EMAIL_RE.replace(s, "<email>")
// YouTube channel / playlist / bare video ids (watch behavior). // YouTube channel + playlist ids (watch behavior).
// KEPT on the dogfood profile — the signal we triage outages on.
if (!keepIdentifiers) {
s = CHANNEL_ID_RE.replace(s, "<channelId>") s = CHANNEL_ID_RE.replace(s, "<channelId>")
s = PLAYLIST_ID_RE.replace(s, "<playlistId>") s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged // Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
@ -209,26 +176,9 @@ object LogDump {
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m -> s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
} }
}
// Long high-entropy runs (hashes, visitor data, unlabeled // Long high-entropy runs (hashes, visitor data, unlabeled
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit. Scrubbed in // tokens): 20+ [A-Za-z0-9_-] chars containing a digit.
// BOTH profiles — visitorData & friends are creds, not signal. On the s = LONG_TOKEN_RE.replace(s, "<token>")
// dogfood profile a token that is exactly a channel/playlist id
// (UC…, PL…/UU…/LL…/RD…/OLAK5uy_…) is KEPT — those ride past the
// skipped id passes above but LONG_TOKEN would otherwise redact them,
// and they're the same triage signal as the video id we keep. Every
// other high-entropy run (visitorData, hashes) still redacts.
s = if (keepIdentifiers) {
LONG_TOKEN_RE.replace(s) { m ->
if (CHANNEL_ID_RE.matches(m.value) || PLAYLIST_ID_RE.matches(m.value)) {
m.value
} else {
"<token>"
}
}
} else {
LONG_TOKEN_RE.replace(s, "<token>")
}
// IP addresses (v4 + v6 — v6 patterns are shaped so threadtime // IP addresses (v4 + v6 — v6 patterns are shaped so threadtime
// HH:MM:SS timestamps can never match). // HH:MM:SS timestamps can never match).
s = IPV4_RE.replace(s, "<ip>") s = IPV4_RE.replace(s, "<ip>")
@ -300,17 +250,7 @@ object LogDump {
private val IPV6_FULL_RE = Regex( private val IPV6_FULL_RE = Regex(
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){5,7}[0-9A-Fa-f]{1,4}(?![0-9A-Fa-f:])""", """(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){5,7}[0-9A-Fa-f]{1,4}(?![0-9A-Fa-f:])""",
) )
// Compressed form requires a literal `::`. The head is EITHER one-or-more
// `hex:` groups (`fe80::…`, `2001:db8::…`) OR empty for a leading `::`
// (`::1`); the tail after `::` is now REQUIRED (>=1 hex group). That kills
// the old false positive where a Rust module path — a lone hex letter + `::`
// + a non-hex word, e.g. `strawcore::stream` — got scrubbed to
// `strawcor<ip>stream`. The trailing lookahead rejects a following word char
// so `stream::foo` (foo starts with hex `f`) can't clip `::f` either. Real
// compressed IPv6 (`::1`, `fe80::1`, `2001:db8::1`) still scrubs; a bare
// `fe80::`-with-nothing-after is intentionally let through (rare in logs,
// and cheaper than re-opening the false-positive hole).
private val IPV6_COMPRESSED_RE = Regex( private val IPV6_COMPRESSED_RE = Regex(
"""(?<![0-9A-Fa-f:])(?:(?:[0-9A-Fa-f]{1,4}:){1,6}|:):[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*(?![0-9A-Za-z:_-])""", """(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,5})?(?![0-9A-Fa-f:])""",
) )
} }

View file

@ -1,132 +0,0 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Unit tests for LogDump's two scrub profiles.
*
* These are pure-JVM tests over LogDump.scrubLine / scrubLineDogfood the
* regexes have no Android dependency, so they run under the standard
* `:strawApp:testDebugUnitTest` task with junit on the test classpath.
*
* NOTE: this module had no test source set before; running these needs
* `testImplementation("junit:junit:4.13.2")` in strawApp/build.gradle.kts
* and a CI step that invokes the test task (build.yml only runs
* assembleDebug today, which does NOT compile/run this source set).
*/
package com.sulkta.straw.util
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
class LogDumpScrubTest {
// ---- B1: IPv6-compressed `::` false positive -------------------------
@Test
fun rustPathWithDoubleColonSurvives() {
// The bug: `strawcore::stream` was scrubbed to `strawcor<ip>stream`
// because `e::` read as a compressed IPv6. It must now pass through
// untouched in BOTH profiles.
assertEquals("strawcore::stream", LogDump.scrubLine("strawcore::stream"))
assertEquals("strawcore::stream", LogDump.scrubLineDogfood("strawcore::stream"))
// A three-segment path — `::foo` must not clip `::f` (foo starts with
// the hex letter `f`).
assertEquals("strawcore::stream::foo", LogDump.scrubLine("strawcore::stream::foo"))
// Embedded in a realistic log line.
val line = "W strawcore: extracting strawcore::stream::foo failed"
assertEquals(line, LogDump.scrubLine(line))
}
@Test
fun realCompressedIpv6StillScrubs() {
// Loopback `::1` (a leading `::` — the OLD regex actually missed this),
// link-local, and documentation-range compressed forms all redact.
assertEquals("<ip>", LogDump.scrubLine("::1"))
assertEquals("<ip>", LogDump.scrubLine("fe80::1"))
assertEquals("<ip>", LogDump.scrubLine("2001:db8::1"))
assertEquals("<ip>", LogDump.scrubLine("2604:2dc0::1"))
// In-context, too.
assertTrue(LogDump.scrubLine("bound peer fe80::1 up").contains("<ip>"))
assertFalse(LogDump.scrubLine("bound peer fe80::1 up").contains("fe80"))
}
// ---- B2: dogfood profile KEEPS diagnostic identifiers ----------------
@Test
fun dogfoodKeepsBareVideoIdFullScrubs() {
val id = "dQw4w9WgXcQ"
assertEquals(id, LogDump.scrubLineDogfood(id))
assertEquals("<videoId>", LogDump.scrubLine(id))
}
@Test
fun dogfoodKeepsYoutubeShortLinkFullScrubs() {
val line = "open youtu.be/dQw4w9WgXcQ now"
// Dogfood: the youtu.be/<id> stays — the id is the diagnostic signal.
assertEquals(line, LogDump.scrubLineDogfood(line))
// Full: the schemeless YT link is redacted.
assertTrue(LogDump.scrubLine(line).contains("<scrubbed>"))
assertFalse(LogDump.scrubLine(line).contains("dQw4w9WgXcQ"))
}
@Test
fun dogfoodKeepsChannelAndPlaylistIdFullScrubs() {
// These are ≥20 chars so LONG_TOKEN would otherwise redact them even
// on the dogfood profile; the channel/playlist exemption keeps them.
val channelId = "UCuAXFkgsw1L7xaCfnd5JJOw" // UC + 22, has digits
val playlistId = "PLbpi6ZahtOH6Blw3RGYpWkSByi_T7Rygb"
assertEquals(channelId, LogDump.scrubLineDogfood(channelId))
assertEquals(playlistId, LogDump.scrubLineDogfood(playlistId))
// Full profile redacts them to their labelled placeholders.
assertEquals("<channelId>", LogDump.scrubLine(channelId))
assertEquals("<playlistId>", LogDump.scrubLine(playlistId))
}
@Test
fun dogfoodStillScrubsGenericHighEntropyToken() {
// visitorData-shaped: ≥20 chars with a digit, NOT a channel/playlist
// prefix → still `<token>` on the dogfood profile (it's a cred).
val visitorData = "CgtVQzEyMzQ1Njc4OTBhYg"
assertEquals("<token>", LogDump.scrubLineDogfood(visitorData))
assertEquals("<token>", LogDump.scrubLine(visitorData))
}
// ---- B2: dogfood profile STILL scrubs real credentials / PII ---------
@Test
fun dogfoodStillScrubsSignedGooglevideoUrl() {
val line =
"url=https://r5---sn-abc.googlevideo.com/videoplayback?expire=99&sig=DEADBEEFSIG&pot=SECRETPOT"
val out = LogDump.scrubLineDogfood(line)
assertTrue(out.contains("<scrubbed>"))
assertFalse(out.contains("DEADBEEFSIG"))
assertFalse(out.contains("SECRETPOT"))
}
@Test
fun dogfoodStillScrubsBearerToken() {
val out = LogDump.scrubLineDogfood("hdr Authorization: Bearer ya29.A0ARLongTokenValue123")
assertTrue(out.contains("<scrubbed>"))
assertFalse(out.contains("ya29"))
}
@Test
fun dogfoodStillScrubsSignedParams() {
val out = LogDump.scrubLineDogfood("req q&sig=DEADBEEF123&pot=POTVAL456 done")
assertTrue(out.contains("sig=<scrubbed>"))
assertTrue(out.contains("pot=<scrubbed>"))
assertFalse(out.contains("DEADBEEF123"))
assertFalse(out.contains("POTVAL456"))
}
@Test
fun dogfoodStillScrubsIpAddresses() {
val out = LogDump.scrubLineDogfood("peer 203.0.113.7 and fe80::1 up")
assertTrue(out.contains("<ip>"))
assertFalse(out.contains("203.0.113.7"))
assertFalse(out.contains("fe80"))
}
}