diag: expand dogfood logging — lighter scrub + always-log failures + wrapper catch-all
All checks were successful
build-apk / build-and-publish (push) Successful in 8m2s

Pairs with strawcore's new extractor instrumentation to make "Send logs to Kayos"
actually diagnosable. Today's bot-wall outage produced a log dump that was 90%
framework noise, one mangled strawcore line, and zero failure signal.

- LogDump: split the scrubber into a FULL profile (share-sheet export + on-screen
  error strings — unchanged, still over-redacts) and a lighter DOGFOOD profile
  (the logs.sulkta.com ingest path, our own infra) that KEEPS bare YouTube ids
  (video/channel/playlist/youtu.be — the triage signal) while still scrubbing
  every real credential: signed googlevideo URLs, bearer/cookie/token headers,
  sig/pot/n/cpn params, URL queries, emails, high-entropy tokens (visitorData),
  and IPs. LogShipper uses the dogfood profile.
- Fixed the IPv6-compressed regex that mangled Rust module paths
  (`strawcore::stream` -> `strawcor<ip>stream`) and, as a bonus, a latent
  false-negative where `::1` was never scrubbed.
- Always-log extraction + playback failures (strawLogI) so a user-visible failure
  always lands in the ring.
- Wrapper (rust/strawcore): one catch-all WARN in run_extract that traces EVERY
  extraction failure system-wide, and raise android_logger Info->Debug so the
  extractor's DEBUG tier is live in the dogfood build.
- Added a JVM unit test for the scrubber (not yet wired into CI — the repo has no
  test source set; follow-up).
This commit is contained in:
Cobb 2026-08-06 08:17:15 -07:00
parent 06e27775fe
commit 4d3ddf502f
9 changed files with 225 additions and 31 deletions

View file

@ -26,6 +26,7 @@ import com.sulkta.straw.net.SponsorBlockClient
import com.sulkta.straw.feature.search.StreamItem
import com.sulkta.straw.util.isAllowedYtUrl
import com.sulkta.straw.util.runCatchingCancellable
import com.sulkta.straw.util.strawLogI
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
@ -330,12 +331,18 @@ class VideoDetailViewModel : ViewModel() {
} catch (t: Throwable) {
if (t is CancellationException) throw t
if (_ui.value.loadedUrl != streamUrl) return@launch
// FULL scrub — this reason is rendered to the UI below (and so
// is screenshot-reachable). The always-log line seeds the
// dogfood ring with the user-visible extraction failure; the
// ring's own (lighter) scrub applies at send time.
val reason = com.sulkta.straw.util.LogDump.scrubLine(
t.message ?: t.javaClass.simpleName,
)
strawLogI("StrawDetail", "extraction failed: $reason")
_ui.update {
VideoDetailUiState(
loading = false,
error = com.sulkta.straw.util.LogDump.scrubLine(
t.message ?: t.javaClass.simpleName,
),
error = reason,
loadedUrl = streamUrl,
)
}

View file

@ -13,8 +13,11 @@
* * manual the Settings Diagnostics "Send logs to Kayos" button.
*
* PRIVACY: every line including the crash stacktrace goes through
* LogDump.scrubLine before leaving the device; the server re-scrubs,
* but the phone is the primary gate. device_id is a per-install random
* LogDump.scrubLineDogfood before leaving the device (the lighter
* profile for our own logs.sulkta.com: it still redacts every
* credential/URL/token, but keeps bare video/channel/playlist ids as
* outage-triage signal); the server re-scrubs, but the phone is the
* primary gate. device_id is a per-install random
* UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole
* feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was
* injected (see strawApp/build.gradle.kts) for everyone else
@ -172,8 +175,11 @@ object LogShipper {
throwable.printStackTrace(PrintWriter(it))
}.toString()
.split('\n')
// Dogfood profile: this whole shipper targets our own
// logs.sulkta.com, so keep diagnostic ids while still
// scrubbing every credential/URL/token the trace may carry.
.take(400)
.map(LogDump::scrubLine)
.map(LogDump::scrubLineDogfood)
val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES)
postLines(
context = context,

View file

@ -105,6 +105,7 @@ import com.sulkta.straw.data.Settings
import com.sulkta.straw.feature.detail.VideoDetailBody
import com.sulkta.straw.feature.detail.VideoDetailViewModel
import com.sulkta.straw.util.LogDump
import com.sulkta.straw.util.strawLogI
import kotlinx.coroutines.launch
// Snappy, no overshoot. A spring adapts its speed to the remaining
@ -482,7 +483,12 @@ private fun InlinePlayerSurface(
val listener = object : Player.Listener {
override fun onPlayerError(error: androidx.media3.common.PlaybackException) {
val raw = error.message ?: "(no message)"
playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
// FULL scrub — this is rendered in the on-screen error banner.
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
playbackError = scrubbed
// Always-log so the dogfood ring keeps the user-visible
// playback failure (the ring's own scrub applies at send time).
strawLogI("StrawPlayer", "playback error: $scrubbed")
NowPlaying.clear()
}
}

View file

@ -132,7 +132,12 @@ fun PlayerScreen(
// string — visible in the on-screen error banner and
// a screenshot away from being shared.
val raw = error.message ?: "(no message)"
playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
// FULL scrub — this is rendered in the on-screen error banner.
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
playbackError = scrubbed
// Always-log so the dogfood ring keeps the user-visible
// playback failure (the ring's own scrub applies at send time).
strawLogI("StrawPlayer", "playback error: $scrubbed")
// Also clear NowPlaying so the minibar doesn't keep
// claiming a dead session is loaded.
NowPlaying.clear()

View file

@ -122,7 +122,9 @@ object LogDump {
}
// Scrub only the retained window — cheaper than scrubbing lines
// that get dropped, and nothing raw ever leaves this function.
return ring.map(::scrubLine)
// This is the dogfood ingest path (LogShipper → logs.sulkta.com, our
// own infra), so use the lighter profile that keeps diagnostic ids.
return ring.map(::scrubLineDogfood)
}
/**
@ -132,10 +134,13 @@ object LogDump {
* approach catches every documented leak vector at zero
* allocation cost.
*
* BIAS: over-redaction. These lines leave the device (share sheet
* + LogShipper logs.sulkta.com), so a false positive costs a
* little debug signal while a false negative leaks what someone
* watched. Anything URL-, token-, id-, or address-shaped goes.
* BIAS: over-redaction. This FULL profile backs the share-sheet
* export and every on-screen error string, which can be handed to a
* chooser app or screenshotted, so a false positive costs a little
* debug signal while a false negative leaks what someone watched.
* Anything URL-, token-, id-, or address-shaped goes. (The dogfood
* ingest path to our own logs.sulkta.com uses the lighter
* [scrubLineDogfood] instead it keeps bare identifiers.)
*
* Public so error-handler call sites (PlayerScreen / VideoDetail
* `playbackError`) can scrub Media3's `PlaybackException.message`
@ -143,7 +148,30 @@ object LogDump {
* request URI for HttpDataSource exceptions, which would otherwise
* be a leak via screenshot.
*/
fun scrubLine(line: String): String {
fun scrubLine(line: String): String = scrub(line, keepIdentifiers = false)
/**
* DOGFOOD profile for the logs.sulkta.com ingest path ONLY (our own
* infra, driven by [LogShipper]). Scrubs every real credential / PII
* vector exactly as [scrubLine] does (googlevideo URLs, bearer tokens,
* cred headers/params, signed params, URL tails, emails, high-entropy
* tokens, IPs), but KEEPS the bare YouTube identifiers 11-char video
* ids, channel/playlist ids, schemeless youtu.be links because those
* are the diagnostic signal we triage an outage on, and they only ever
* reach a server we run. NEVER use this for the share-sheet export or a
* string rendered to the UI; those keep the FULL [scrubLine] profile.
*/
fun scrubLineDogfood(line: String): String = scrub(line, keepIdentifiers = true)
/**
* Shared scrub core. [keepIdentifiers] = true selects the lighter
* DOGFOOD profile: it SKIPS the four identifier replacements (schemeless
* youtu.be, channel id, playlist id, bare video id) so those survive,
* while every credential / high-entropy / IP vector is still redacted in
* BOTH profiles. Step ORDER is unchanged from the original full pass, so
* `keepIdentifiers = false` is byte-for-byte the old behavior.
*/
private fun scrub(line: String, keepIdentifiers: Boolean): String {
var s = line
// Pre-signed googlevideo URLs: keep the host label visible, drop
// host prefix + path + query (session-bound streaming creds).
@ -162,22 +190,29 @@ object LogDump {
// /vi/<id>/ thumbnails, search terms, etc. live.
s = URL_TAIL_RE.replace(s, "$1/<scrubbed>")
// Schemeless YT links pasted into log messages ("youtu.be/xyz").
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
// KEPT on the dogfood profile — the id is diagnostic signal.
if (!keepIdentifiers) {
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
}
// Emails.
s = EMAIL_RE.replace(s, "<email>")
// YouTube channel + playlist ids (watch behavior).
s = CHANNEL_ID_RE.replace(s, "<channelId>")
s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
// outside any URL (rustypipe/strawcore do this). Requiring a
// digit/_/- inside the run spares ordinary 11-letter words;
// real ids virtually always contain one. Pure-alpha ids are
// still caught earlier when keyed (v=…) or inside a URL.
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
// YouTube channel / playlist / bare video ids (watch behavior).
// KEPT on the dogfood profile — the signal we triage outages on.
if (!keepIdentifiers) {
s = CHANNEL_ID_RE.replace(s, "<channelId>")
s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
// outside any URL (rustypipe/strawcore do this). Requiring a
// digit/_/- inside the run spares ordinary 11-letter words;
// real ids virtually always contain one. Pure-alpha ids are
// still caught earlier when keyed (v=…) or inside a URL.
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
}
}
// Long high-entropy runs (hashes, visitor data, unlabeled
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit.
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit. Scrubbed
// in BOTH profiles — visitorData & friends are creds, not signal.
s = LONG_TOKEN_RE.replace(s, "<token>")
// IP addresses (v4 + v6 — v6 patterns are shaped so threadtime
// HH:MM:SS timestamps can never match).
@ -250,7 +285,17 @@ object LogDump {
private val IPV6_FULL_RE = Regex(
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){5,7}[0-9A-Fa-f]{1,4}(?![0-9A-Fa-f:])""",
)
// Compressed form requires a literal `::`. The head is EITHER one-or-more
// `hex:` groups (`fe80::…`, `2001:db8::…`) OR empty for a leading `::`
// (`::1`); the tail after `::` is now REQUIRED (>=1 hex group). That kills
// the old false positive where a Rust module path — a lone hex letter + `::`
// + a non-hex word, e.g. `strawcore::stream` — got scrubbed to
// `strawcor<ip>stream`. The trailing lookahead rejects a following word char
// so `stream::foo` (foo starts with hex `f`) can't clip `::f` either. Real
// compressed IPv6 (`::1`, `fe80::1`, `2001:db8::1`) still scrubs; a bare
// `fe80::`-with-nothing-after is intentionally let through (rare in logs,
// and cheaper than re-opening the false-positive hole).
private val IPV6_COMPRESSED_RE = Regex(
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,5})?(?![0-9A-Fa-f:])""",
"""(?<![0-9A-Fa-f:])(?:(?:[0-9A-Fa-f]{1,4}:){1,6}|:):[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*(?![0-9A-Za-z:_-])""",
)
}

View file

@ -0,0 +1,110 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta
* SPDX-License-Identifier: GPL-3.0-or-later
*
* Unit tests for LogDump's two scrub profiles.
*
* These are pure-JVM tests over LogDump.scrubLine / scrubLineDogfood the
* regexes have no Android dependency, so they run under the standard
* `:strawApp:testDebugUnitTest` task with junit on the test classpath.
*
* NOTE: this module had no test source set before; running these needs
* `testImplementation("junit:junit:4.13.2")` in strawApp/build.gradle.kts
* and a CI step that invokes the test task (build.yml only runs
* assembleDebug today, which does NOT compile/run this source set).
*/
package com.sulkta.straw.util
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
class LogDumpScrubTest {
// ---- B1: IPv6-compressed `::` false positive -------------------------
@Test
fun rustPathWithDoubleColonSurvives() {
// The bug: `strawcore::stream` was scrubbed to `strawcor<ip>stream`
// because `e::` read as a compressed IPv6. It must now pass through
// untouched in BOTH profiles.
assertEquals("strawcore::stream", LogDump.scrubLine("strawcore::stream"))
assertEquals("strawcore::stream", LogDump.scrubLineDogfood("strawcore::stream"))
// A three-segment path — `::foo` must not clip `::f` (foo starts with
// the hex letter `f`).
assertEquals("strawcore::stream::foo", LogDump.scrubLine("strawcore::stream::foo"))
// Embedded in a realistic log line.
val line = "W strawcore: extracting strawcore::stream::foo failed"
assertEquals(line, LogDump.scrubLine(line))
}
@Test
fun realCompressedIpv6StillScrubs() {
// Loopback `::1` (a leading `::` — the OLD regex actually missed this),
// link-local, and documentation-range compressed forms all redact.
assertEquals("<ip>", LogDump.scrubLine("::1"))
assertEquals("<ip>", LogDump.scrubLine("fe80::1"))
assertEquals("<ip>", LogDump.scrubLine("2001:db8::1"))
assertEquals("<ip>", LogDump.scrubLine("2604:2dc0::1"))
// In-context, too.
assertTrue(LogDump.scrubLine("bound peer fe80::1 up").contains("<ip>"))
assertFalse(LogDump.scrubLine("bound peer fe80::1 up").contains("fe80"))
}
// ---- B2: dogfood profile KEEPS diagnostic identifiers ----------------
@Test
fun dogfoodKeepsBareVideoIdFullScrubs() {
val id = "dQw4w9WgXcQ"
assertEquals(id, LogDump.scrubLineDogfood(id))
assertEquals("<videoId>", LogDump.scrubLine(id))
}
@Test
fun dogfoodKeepsYoutubeShortLinkFullScrubs() {
val line = "open youtu.be/dQw4w9WgXcQ now"
// Dogfood: the youtu.be/<id> stays — the id is the diagnostic signal.
assertEquals(line, LogDump.scrubLineDogfood(line))
// Full: the schemeless YT link is redacted.
assertTrue(LogDump.scrubLine(line).contains("<scrubbed>"))
assertFalse(LogDump.scrubLine(line).contains("dQw4w9WgXcQ"))
}
// ---- B2: dogfood profile STILL scrubs real credentials / PII ---------
@Test
fun dogfoodStillScrubsSignedGooglevideoUrl() {
val line =
"url=https://r5---sn-abc.googlevideo.com/videoplayback?expire=99&sig=DEADBEEFSIG&pot=SECRETPOT"
val out = LogDump.scrubLineDogfood(line)
assertTrue(out.contains("<scrubbed>"))
assertFalse(out.contains("DEADBEEFSIG"))
assertFalse(out.contains("SECRETPOT"))
}
@Test
fun dogfoodStillScrubsBearerToken() {
val out = LogDump.scrubLineDogfood("hdr Authorization: Bearer ya29.A0ARLongTokenValue123")
assertTrue(out.contains("<scrubbed>"))
assertFalse(out.contains("ya29"))
}
@Test
fun dogfoodStillScrubsSignedParams() {
val out = LogDump.scrubLineDogfood("req q&sig=DEADBEEF123&pot=POTVAL456 done")
assertTrue(out.contains("sig=<scrubbed>"))
assertTrue(out.contains("pot=<scrubbed>"))
assertFalse(out.contains("DEADBEEF123"))
assertFalse(out.contains("POTVAL456"))
}
@Test
fun dogfoodStillScrubsIpAddresses() {
val out = LogDump.scrubLineDogfood("peer 203.0.113.7 and fe80::1 up")
assertTrue(out.contains("<ip>"))
assertFalse(out.contains("203.0.113.7"))
assertFalse(out.contains("fe80"))
}
}