diag: expand dogfood logging — lighter scrub + always-log failures + wrapper catch-all
All checks were successful
build-apk / build-and-publish (push) Successful in 8m2s
All checks were successful
build-apk / build-and-publish (push) Successful in 8m2s
Pairs with strawcore's new extractor instrumentation to make "Send logs to Kayos" actually diagnosable. Today's bot-wall outage produced a log dump that was 90% framework noise, one mangled strawcore line, and zero failure signal. - LogDump: split the scrubber into a FULL profile (share-sheet export + on-screen error strings — unchanged, still over-redacts) and a lighter DOGFOOD profile (the logs.sulkta.com ingest path, our own infra) that KEEPS bare YouTube ids (video/channel/playlist/youtu.be — the triage signal) while still scrubbing every real credential: signed googlevideo URLs, bearer/cookie/token headers, sig/pot/n/cpn params, URL queries, emails, high-entropy tokens (visitorData), and IPs. LogShipper uses the dogfood profile. - Fixed the IPv6-compressed regex that mangled Rust module paths (`strawcore::stream` -> `strawcor<ip>stream`) and, as a bonus, a latent false-negative where `::1` was never scrubbed. - Always-log extraction + playback failures (strawLogI) so a user-visible failure always lands in the ring. - Wrapper (rust/strawcore): one catch-all WARN in run_extract that traces EVERY extraction failure system-wide, and raise android_logger Info->Debug so the extractor's DEBUG tier is live in the dogfood build. - Added a JVM unit test for the scrubber (not yet wired into CI — the repo has no test source set; follow-up).
This commit is contained in:
parent
06e27775fe
commit
4d3ddf502f
9 changed files with 225 additions and 31 deletions
|
|
@ -26,6 +26,7 @@ import com.sulkta.straw.net.SponsorBlockClient
|
|||
import com.sulkta.straw.feature.search.StreamItem
|
||||
import com.sulkta.straw.util.isAllowedYtUrl
|
||||
import com.sulkta.straw.util.runCatchingCancellable
|
||||
import com.sulkta.straw.util.strawLogI
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
|
|
@ -330,12 +331,18 @@ class VideoDetailViewModel : ViewModel() {
|
|||
} catch (t: Throwable) {
|
||||
if (t is CancellationException) throw t
|
||||
if (_ui.value.loadedUrl != streamUrl) return@launch
|
||||
// FULL scrub — this reason is rendered to the UI below (and so
|
||||
// is screenshot-reachable). The always-log line seeds the
|
||||
// dogfood ring with the user-visible extraction failure; the
|
||||
// ring's own (lighter) scrub applies at send time.
|
||||
val reason = com.sulkta.straw.util.LogDump.scrubLine(
|
||||
t.message ?: t.javaClass.simpleName,
|
||||
)
|
||||
strawLogI("StrawDetail", "extraction failed: $reason")
|
||||
_ui.update {
|
||||
VideoDetailUiState(
|
||||
loading = false,
|
||||
error = com.sulkta.straw.util.LogDump.scrubLine(
|
||||
t.message ?: t.javaClass.simpleName,
|
||||
),
|
||||
error = reason,
|
||||
loadedUrl = streamUrl,
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,8 +13,11 @@
|
|||
* * manual — the Settings → Diagnostics "Send logs to Kayos" button.
|
||||
*
|
||||
* PRIVACY: every line — including the crash stacktrace — goes through
|
||||
* LogDump.scrubLine before leaving the device; the server re-scrubs,
|
||||
* but the phone is the primary gate. device_id is a per-install random
|
||||
* LogDump.scrubLineDogfood before leaving the device (the lighter
|
||||
* profile for our own logs.sulkta.com: it still redacts every
|
||||
* credential/URL/token, but keeps bare video/channel/playlist ids as
|
||||
* outage-triage signal); the server re-scrubs, but the phone is the
|
||||
* primary gate. device_id is a per-install random
|
||||
* UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole
|
||||
* feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was
|
||||
* injected (see strawApp/build.gradle.kts) — for everyone else
|
||||
|
|
@ -172,8 +175,11 @@ object LogShipper {
|
|||
throwable.printStackTrace(PrintWriter(it))
|
||||
}.toString()
|
||||
.split('\n')
|
||||
// Dogfood profile: this whole shipper targets our own
|
||||
// logs.sulkta.com, so keep diagnostic ids while still
|
||||
// scrubbing every credential/URL/token the trace may carry.
|
||||
.take(400)
|
||||
.map(LogDump::scrubLine)
|
||||
.map(LogDump::scrubLineDogfood)
|
||||
val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES)
|
||||
postLines(
|
||||
context = context,
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ import com.sulkta.straw.data.Settings
|
|||
import com.sulkta.straw.feature.detail.VideoDetailBody
|
||||
import com.sulkta.straw.feature.detail.VideoDetailViewModel
|
||||
import com.sulkta.straw.util.LogDump
|
||||
import com.sulkta.straw.util.strawLogI
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
// Snappy, no overshoot. A spring adapts its speed to the remaining
|
||||
|
|
@ -482,7 +483,12 @@ private fun InlinePlayerSurface(
|
|||
val listener = object : Player.Listener {
|
||||
override fun onPlayerError(error: androidx.media3.common.PlaybackException) {
|
||||
val raw = error.message ?: "(no message)"
|
||||
playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
|
||||
// FULL scrub — this is rendered in the on-screen error banner.
|
||||
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
|
||||
playbackError = scrubbed
|
||||
// Always-log so the dogfood ring keeps the user-visible
|
||||
// playback failure (the ring's own scrub applies at send time).
|
||||
strawLogI("StrawPlayer", "playback error: $scrubbed")
|
||||
NowPlaying.clear()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -132,7 +132,12 @@ fun PlayerScreen(
|
|||
// string — visible in the on-screen error banner and
|
||||
// a screenshot away from being shared.
|
||||
val raw = error.message ?: "(no message)"
|
||||
playbackError = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
|
||||
// FULL scrub — this is rendered in the on-screen error banner.
|
||||
val scrubbed = "${error.errorCodeName}: ${LogDump.scrubLine(raw)}"
|
||||
playbackError = scrubbed
|
||||
// Always-log so the dogfood ring keeps the user-visible
|
||||
// playback failure (the ring's own scrub applies at send time).
|
||||
strawLogI("StrawPlayer", "playback error: $scrubbed")
|
||||
// Also clear NowPlaying so the minibar doesn't keep
|
||||
// claiming a dead session is loaded.
|
||||
NowPlaying.clear()
|
||||
|
|
|
|||
|
|
@ -122,7 +122,9 @@ object LogDump {
|
|||
}
|
||||
// Scrub only the retained window — cheaper than scrubbing lines
|
||||
// that get dropped, and nothing raw ever leaves this function.
|
||||
return ring.map(::scrubLine)
|
||||
// This is the dogfood ingest path (LogShipper → logs.sulkta.com, our
|
||||
// own infra), so use the lighter profile that keeps diagnostic ids.
|
||||
return ring.map(::scrubLineDogfood)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -132,10 +134,13 @@ object LogDump {
|
|||
* approach catches every documented leak vector at zero
|
||||
* allocation cost.
|
||||
*
|
||||
* BIAS: over-redaction. These lines leave the device (share sheet
|
||||
* + LogShipper → logs.sulkta.com), so a false positive costs a
|
||||
* little debug signal while a false negative leaks what someone
|
||||
* watched. Anything URL-, token-, id-, or address-shaped goes.
|
||||
* BIAS: over-redaction. This FULL profile backs the share-sheet
|
||||
* export and every on-screen error string, which can be handed to a
|
||||
* chooser app or screenshotted, so a false positive costs a little
|
||||
* debug signal while a false negative leaks what someone watched.
|
||||
* Anything URL-, token-, id-, or address-shaped goes. (The dogfood
|
||||
* ingest path to our own logs.sulkta.com uses the lighter
|
||||
* [scrubLineDogfood] instead — it keeps bare identifiers.)
|
||||
*
|
||||
* Public so error-handler call sites (PlayerScreen / VideoDetail
|
||||
* `playbackError`) can scrub Media3's `PlaybackException.message`
|
||||
|
|
@ -143,7 +148,30 @@ object LogDump {
|
|||
* request URI for HttpDataSource exceptions, which would otherwise
|
||||
* be a leak via screenshot.
|
||||
*/
|
||||
fun scrubLine(line: String): String {
|
||||
fun scrubLine(line: String): String = scrub(line, keepIdentifiers = false)
|
||||
|
||||
/**
|
||||
* DOGFOOD profile — for the logs.sulkta.com ingest path ONLY (our own
|
||||
* infra, driven by [LogShipper]). Scrubs every real credential / PII
|
||||
* vector exactly as [scrubLine] does (googlevideo URLs, bearer tokens,
|
||||
* cred headers/params, signed params, URL tails, emails, high-entropy
|
||||
* tokens, IPs), but KEEPS the bare YouTube identifiers — 11-char video
|
||||
* ids, channel/playlist ids, schemeless youtu.be links — because those
|
||||
* are the diagnostic signal we triage an outage on, and they only ever
|
||||
* reach a server we run. NEVER use this for the share-sheet export or a
|
||||
* string rendered to the UI; those keep the FULL [scrubLine] profile.
|
||||
*/
|
||||
fun scrubLineDogfood(line: String): String = scrub(line, keepIdentifiers = true)
|
||||
|
||||
/**
|
||||
* Shared scrub core. [keepIdentifiers] = true selects the lighter
|
||||
* DOGFOOD profile: it SKIPS the four identifier replacements (schemeless
|
||||
* youtu.be, channel id, playlist id, bare video id) so those survive,
|
||||
* while every credential / high-entropy / IP vector is still redacted in
|
||||
* BOTH profiles. Step ORDER is unchanged from the original full pass, so
|
||||
* `keepIdentifiers = false` is byte-for-byte the old behavior.
|
||||
*/
|
||||
private fun scrub(line: String, keepIdentifiers: Boolean): String {
|
||||
var s = line
|
||||
// Pre-signed googlevideo URLs: keep the host label visible, drop
|
||||
// host prefix + path + query (session-bound streaming creds).
|
||||
|
|
@ -162,22 +190,29 @@ object LogDump {
|
|||
// /vi/<id>/ thumbnails, search terms, etc. live.
|
||||
s = URL_TAIL_RE.replace(s, "$1/<scrubbed>")
|
||||
// Schemeless YT links pasted into log messages ("youtu.be/xyz").
|
||||
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
|
||||
// KEPT on the dogfood profile — the id is diagnostic signal.
|
||||
if (!keepIdentifiers) {
|
||||
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
|
||||
}
|
||||
// Emails.
|
||||
s = EMAIL_RE.replace(s, "<email>")
|
||||
// YouTube channel + playlist ids (watch behavior).
|
||||
s = CHANNEL_ID_RE.replace(s, "<channelId>")
|
||||
s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
|
||||
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
|
||||
// outside any URL (rustypipe/strawcore do this). Requiring a
|
||||
// digit/_/- inside the run spares ordinary 11-letter words;
|
||||
// real ids virtually always contain one. Pure-alpha ids are
|
||||
// still caught earlier when keyed (v=…) or inside a URL.
|
||||
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
|
||||
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
|
||||
// YouTube channel / playlist / bare video ids (watch behavior).
|
||||
// KEPT on the dogfood profile — the signal we triage outages on.
|
||||
if (!keepIdentifiers) {
|
||||
s = CHANNEL_ID_RE.replace(s, "<channelId>")
|
||||
s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
|
||||
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
|
||||
// outside any URL (rustypipe/strawcore do this). Requiring a
|
||||
// digit/_/- inside the run spares ordinary 11-letter words;
|
||||
// real ids virtually always contain one. Pure-alpha ids are
|
||||
// still caught earlier when keyed (v=…) or inside a URL.
|
||||
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
|
||||
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
|
||||
}
|
||||
}
|
||||
// Long high-entropy runs (hashes, visitor data, unlabeled
|
||||
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit.
|
||||
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit. Scrubbed
|
||||
// in BOTH profiles — visitorData & friends are creds, not signal.
|
||||
s = LONG_TOKEN_RE.replace(s, "<token>")
|
||||
// IP addresses (v4 + v6 — v6 patterns are shaped so threadtime
|
||||
// HH:MM:SS timestamps can never match).
|
||||
|
|
@ -250,7 +285,17 @@ object LogDump {
|
|||
private val IPV6_FULL_RE = Regex(
|
||||
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){5,7}[0-9A-Fa-f]{1,4}(?![0-9A-Fa-f:])""",
|
||||
)
|
||||
// Compressed form requires a literal `::`. The head is EITHER one-or-more
|
||||
// `hex:` groups (`fe80::…`, `2001:db8::…`) OR empty for a leading `::`
|
||||
// (`::1`); the tail after `::` is now REQUIRED (>=1 hex group). That kills
|
||||
// the old false positive where a Rust module path — a lone hex letter + `::`
|
||||
// + a non-hex word, e.g. `strawcore::stream` — got scrubbed to
|
||||
// `strawcor<ip>stream`. The trailing lookahead rejects a following word char
|
||||
// so `stream::foo` (foo starts with hex `f`) can't clip `::f` either. Real
|
||||
// compressed IPv6 (`::1`, `fe80::1`, `2001:db8::1`) still scrubs; a bare
|
||||
// `fe80::`-with-nothing-after is intentionally let through (rare in logs,
|
||||
// and cheaper than re-opening the false-positive hole).
|
||||
private val IPV6_COMPRESSED_RE = Regex(
|
||||
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,5})?(?![0-9A-Fa-f:])""",
|
||||
"""(?<![0-9A-Fa-f:])(?:(?:[0-9A-Fa-f]{1,4}:){1,6}|:):[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*(?![0-9A-Za-z:_-])""",
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
*
|
||||
* Unit tests for LogDump's two scrub profiles.
|
||||
*
|
||||
* These are pure-JVM tests over LogDump.scrubLine / scrubLineDogfood — the
|
||||
* regexes have no Android dependency, so they run under the standard
|
||||
* `:strawApp:testDebugUnitTest` task with junit on the test classpath.
|
||||
*
|
||||
* NOTE: this module had no test source set before; running these needs
|
||||
* `testImplementation("junit:junit:4.13.2")` in strawApp/build.gradle.kts
|
||||
* and a CI step that invokes the test task (build.yml only runs
|
||||
* assembleDebug today, which does NOT compile/run this source set).
|
||||
*/
|
||||
|
||||
package com.sulkta.straw.util
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
|
||||
class LogDumpScrubTest {
|
||||
|
||||
// ---- B1: IPv6-compressed `::` false positive -------------------------
|
||||
|
||||
@Test
|
||||
fun rustPathWithDoubleColonSurvives() {
|
||||
// The bug: `strawcore::stream` was scrubbed to `strawcor<ip>stream`
|
||||
// because `e::` read as a compressed IPv6. It must now pass through
|
||||
// untouched in BOTH profiles.
|
||||
assertEquals("strawcore::stream", LogDump.scrubLine("strawcore::stream"))
|
||||
assertEquals("strawcore::stream", LogDump.scrubLineDogfood("strawcore::stream"))
|
||||
// A three-segment path — `::foo` must not clip `::f` (foo starts with
|
||||
// the hex letter `f`).
|
||||
assertEquals("strawcore::stream::foo", LogDump.scrubLine("strawcore::stream::foo"))
|
||||
// Embedded in a realistic log line.
|
||||
val line = "W strawcore: extracting strawcore::stream::foo failed"
|
||||
assertEquals(line, LogDump.scrubLine(line))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun realCompressedIpv6StillScrubs() {
|
||||
// Loopback `::1` (a leading `::` — the OLD regex actually missed this),
|
||||
// link-local, and documentation-range compressed forms all redact.
|
||||
assertEquals("<ip>", LogDump.scrubLine("::1"))
|
||||
assertEquals("<ip>", LogDump.scrubLine("fe80::1"))
|
||||
assertEquals("<ip>", LogDump.scrubLine("2001:db8::1"))
|
||||
assertEquals("<ip>", LogDump.scrubLine("2604:2dc0::1"))
|
||||
// In-context, too.
|
||||
assertTrue(LogDump.scrubLine("bound peer fe80::1 up").contains("<ip>"))
|
||||
assertFalse(LogDump.scrubLine("bound peer fe80::1 up").contains("fe80"))
|
||||
}
|
||||
|
||||
// ---- B2: dogfood profile KEEPS diagnostic identifiers ----------------
|
||||
|
||||
@Test
|
||||
fun dogfoodKeepsBareVideoIdFullScrubs() {
|
||||
val id = "dQw4w9WgXcQ"
|
||||
assertEquals(id, LogDump.scrubLineDogfood(id))
|
||||
assertEquals("<videoId>", LogDump.scrubLine(id))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dogfoodKeepsYoutubeShortLinkFullScrubs() {
|
||||
val line = "open youtu.be/dQw4w9WgXcQ now"
|
||||
// Dogfood: the youtu.be/<id> stays — the id is the diagnostic signal.
|
||||
assertEquals(line, LogDump.scrubLineDogfood(line))
|
||||
// Full: the schemeless YT link is redacted.
|
||||
assertTrue(LogDump.scrubLine(line).contains("<scrubbed>"))
|
||||
assertFalse(LogDump.scrubLine(line).contains("dQw4w9WgXcQ"))
|
||||
}
|
||||
|
||||
// ---- B2: dogfood profile STILL scrubs real credentials / PII ---------
|
||||
|
||||
@Test
|
||||
fun dogfoodStillScrubsSignedGooglevideoUrl() {
|
||||
val line =
|
||||
"url=https://r5---sn-abc.googlevideo.com/videoplayback?expire=99&sig=DEADBEEFSIG&pot=SECRETPOT"
|
||||
val out = LogDump.scrubLineDogfood(line)
|
||||
assertTrue(out.contains("<scrubbed>"))
|
||||
assertFalse(out.contains("DEADBEEFSIG"))
|
||||
assertFalse(out.contains("SECRETPOT"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dogfoodStillScrubsBearerToken() {
|
||||
val out = LogDump.scrubLineDogfood("hdr Authorization: Bearer ya29.A0ARLongTokenValue123")
|
||||
assertTrue(out.contains("<scrubbed>"))
|
||||
assertFalse(out.contains("ya29"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dogfoodStillScrubsSignedParams() {
|
||||
val out = LogDump.scrubLineDogfood("req q&sig=DEADBEEF123&pot=POTVAL456 done")
|
||||
assertTrue(out.contains("sig=<scrubbed>"))
|
||||
assertTrue(out.contains("pot=<scrubbed>"))
|
||||
assertFalse(out.contains("DEADBEEF123"))
|
||||
assertFalse(out.contains("POTVAL456"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun dogfoodStillScrubsIpAddresses() {
|
||||
val out = LogDump.scrubLineDogfood("peer 203.0.113.7 and fe80::1 up")
|
||||
assertTrue(out.contains("<ip>"))
|
||||
assertFalse(out.contains("203.0.113.7"))
|
||||
assertFalse(out.contains("fe80"))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue