diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 0e9ca3ea0..928c3cc4a 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -88,6 +88,10 @@ jobs: STRAW_KEYSTORE_PASS: android STRAW_KEY_ALIAS: androiddebugkey STRAW_KEY_PASS: android + # Dogfood log-shipping: the app is compile-DISABLED unless this is set + # (empty ⇒ no crash handler, no network, Settings row hidden). Repo + # secret; absent on contributor forks so their builds don't phone home. + STRAW_LOGS_INGEST_TOKEN: ${{ secrets.STRAW_LOGS_INGEST_TOKEN }} # Keep the 4-ABI cross-compile off the container rootfs. CARGO_TARGET_DIR: ${{ github.workspace }}/cargo-target # ionice idle class + low nice so the build yields disk/CPU to the diff --git a/strawApp/build.gradle.kts b/strawApp/build.gradle.kts index ed3b5df02..8d13d14cf 100644 --- a/strawApp/build.gradle.kts +++ b/strawApp/build.gradle.kts @@ -10,6 +10,7 @@ */ import com.android.build.api.dsl.ApplicationExtension +import java.util.Properties plugins { alias(libs.plugins.android.application) @@ -36,6 +37,30 @@ configure { versionCode = STRAW_VERSION_CODE versionName = STRAW_VERSION_NAME resValue("string", "app_name", "Straw") + + // Dogfood log-shipping token (feature/diag/LogShipper.kt). This repo + // is PUBLIC — the token must NEVER be committed, so it's injected at + // build time and defaults to "": + // 1. env var STRAW_LOGS_INGEST_TOKEN — CI: add a Forgejo Actions + // repo secret STRAW_LOGS_INGEST_TOKEN and pass it in the + // "Assemble debug APK" step's env block in + // .forgejo/workflows/build.yml. + // 2. `straw.logs.ingest.token=…` in local.properties (gitignored) + // for local dogfood builds. + // Empty token ⇒ LogShipper is fully disabled (no-op, zero network, + // no Settings row) — contributor builds neither break nor phone home. + val strawLogsToken: String = System.getenv("STRAW_LOGS_INGEST_TOKEN") + ?: run { + val props = Properties() + val f = rootProject.file("local.properties") + if (f.exists()) f.inputStream().use { props.load(it) } + props.getProperty("straw.logs.ingest.token") + } + ?: "" + // Escape so a pathological token can't break out of the generated + // String literal in BuildConfig.java. + val escapedToken = strawLogsToken.replace("\\", "\\\\").replace("\"", "\\\"") + buildConfigField("String", "STRAW_LOGS_TOKEN", "\"$escapedToken\"") } // Explicit signing so CI / release builds reuse ONE keystore instead of diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt index e1922b988..e98f4f64a 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt @@ -21,6 +21,7 @@ import com.sulkta.straw.data.SearchCache import com.sulkta.straw.data.Settings import com.sulkta.straw.data.Subscriptions import com.sulkta.straw.feature.dataimport.SettingsImport +import com.sulkta.straw.feature.diag.LogShipper import com.sulkta.straw.feature.feed.FeedRefreshScheduler import com.sulkta.straw.feature.update.UpdateScheduler import com.sulkta.straw.feature.update.runUpdateCheck @@ -90,6 +91,12 @@ class StrawApp : Application(), SingletonImageLoader.Factory { override fun onCreate() { super.onCreate() + // Dogfood crash shipping — wrap the default uncaught-exception + // handler FIRST so even an init crash below gets shipped. The + // wrapper always chains to the previous handler (normal crash + // behavior is untouched) and is a no-op in builds without an + // injected STRAW_LOGS_TOKEN. + LogShipper.installCrashHandler(this) // Path C-7: route Rust `log::*` calls into Android logcat under tag // "strawcore". Without this, every log line emitted from rustypipe / // strawcore is silently dropped, making playback regressions invisible diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt new file mode 100644 index 000000000..5de33c408 --- /dev/null +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt @@ -0,0 +1,230 @@ +/* + * SPDX-FileCopyrightText: 2026 Sulkta + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Dogfood log shipping — POST a scrubbed logcat window to Kayos's + * ingest service (logs.sulkta.com). Two triggers: + * + * * crash — a default-UncaughtExceptionHandler wrapper captures the + * log ring + the throwable's stacktrace, attempts one + * hard-capped best-effort send, then chains to the + * previously-installed handler so the process still dies + * (and Android still shows its crash dialog) normally. + * * manual — the Settings → Diagnostics "Send logs to Kayos" button. + * + * PRIVACY: every line — including the crash stacktrace — goes through + * LogDump.scrubLine before leaving the device; the server re-scrubs, + * but the phone is the primary gate. device_id is a per-install random + * UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole + * feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was + * injected (see strawApp/build.gradle.kts) — for everyone else + * `enabled` is a compile-time false, so R8 strips the network path and + * a contributor build cannot phone home. + * + * DELIVERY: strictly best-effort. Any failure (offline, 401/413/429, + * timeout) is swallowed; nothing here may ever crash or block the app. + */ + +package com.sulkta.straw.feature.diag + +import android.content.Context +import com.sulkta.straw.BuildConfig +import com.sulkta.straw.util.LogDump +import com.sulkta.straw.util.runCatchingCancellable +import java.io.PrintWriter +import java.io.StringWriter +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone +import java.util.UUID +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody + +object LogShipper { + + /** Not a secret — only the bearer token is. */ + private const val INGEST_URL = "https://logs.sulkta.com/ingest" + + /** Server caps: 10k lines / 1 MB body. Stay well under both. */ + private const val MAX_LINES = 2_000 + private const val MAX_BODY_BYTES = 700_000 + + /** Hard cap on how long the crash path may delay process death. */ + private const val CRASH_JOIN_MS = 4_000L + + private const val PREFS = "straw_diag" + private const val KEY_DEVICE_ID = "device_id" + + /** + * Compile-time gate: builds without an injected token have no + * shipping at all — no network, no crash handler, no Settings row. + */ + val enabled: Boolean + get() = BuildConfig.STRAW_LOGS_TOKEN.isNotEmpty() + + // Short timeouts — this is telemetry, not payload. callTimeout is + // the absolute ceiling on the whole request so nothing here can + // hold a WorkManager-style wakelock pattern open. + private val http: OkHttpClient by lazy { + OkHttpClient.Builder() + .connectTimeout(5, TimeUnit.SECONDS) + .writeTimeout(5, TimeUnit.SECONDS) + .readTimeout(5, TimeUnit.SECONDS) + .callTimeout(10, TimeUnit.SECONDS) + .build() + } + + @Serializable + private data class Envelope( + @SerialName("device_id") val deviceId: String, + @SerialName("app_version") val appVersion: String, + @SerialName("sent_at") val sentAt: String, + val reason: String, + val lines: List, + ) + + /** + * Stable per-install id: `straw-`, minted once and + * persisted. Deliberately NOT ANDROID_ID/IMEI — it identifies an + * install for log correlation, never the hardware or the person. + * commit() (not apply()) because both call paths are already on a + * background thread and the crash path dies right after — an + * async write would lose the id and mint a new one per crash. + */ + private fun deviceId(context: Context): String { + val sp = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + sp.getString(KEY_DEVICE_ID, null)?.let { return it } + val id = "straw-${UUID.randomUUID()}" + sp.edit().putString(KEY_DEVICE_ID, id).commit() + return id + } + + private fun isoNowUtc(): String = + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + .apply { timeZone = TimeZone.getTimeZone("UTC") } + .format(Date()) + + /** + * Manual trigger (Settings). Captures + scrubs the log ring and + * POSTs it. Returns true iff the server 2xx'd — the UI shows a + * toast either way; failures never throw. + */ + suspend fun send(context: Context, reason: String): Boolean = + withContext(Dispatchers.IO) { + if (!enabled) return@withContext false + runCatchingCancellable { + postLines( + context = context, + reason = reason, + lines = LogDump.captureScrubbedLinesBlocking(MAX_LINES), + ) + }.getOrDefault(false) + } + + /** + * Wrap the process-default uncaught-exception handler. Ship-then- + * chain: whatever happens in our path (including a throw), the + * previous handler ALWAYS runs, so the app still crashes/reports + * exactly as before. No-op without a token. + */ + fun installCrashHandler(context: Context) { + if (!enabled) return + val appContext = context.applicationContext + val previous = Thread.getDefaultUncaughtExceptionHandler() + Thread.setDefaultUncaughtExceptionHandler { thread, throwable -> + try { + shipCrash(appContext, throwable) + } catch (_: Throwable) { + // Never let diagnostics mask the real crash. + } + previous?.uncaughtException(thread, throwable) + } + } + + @Volatile + private var crashShipStarted = false + + /** + * Crash-path send. The crashing thread may be main (network there + * throws NetworkOnMainThreadException) and is about to die either + * way, so the capture+POST runs on a dedicated worker thread and + * we join with a hard cap — the death is delayed at most + * [CRASH_JOIN_MS], never hung. + */ + private fun shipCrash(context: Context, throwable: Throwable) { + // One attempt per process — also breaks recursion if the ship + // path itself crashes on another thread mid-flight. + if (crashShipStarted) return + crashShipStarted = true + val worker = Thread { + try { + val trace = StringWriter().also { + throwable.printStackTrace(PrintWriter(it)) + }.toString() + .split('\n') + .take(400) + .map(LogDump::scrubLine) + val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES) + postLines( + context = context, + reason = "crash", + lines = ring + "--- uncaught exception ---" + trace, + ) + } catch (_: Throwable) { + // Best-effort only. + } + } + worker.name = "straw-crash-ship" + worker.isDaemon = true + worker.start() + try { + worker.join(CRASH_JOIN_MS) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + + /** + * Build the envelope and POST. `lines` MUST already be scrubbed + * (both call paths run everything through LogDump.scrubLine). + * Trims oldest-first to stay under the server's 1 MB cap — one + * bounded request, no chunking needed at our line cap. + */ + private fun postLines(context: Context, reason: String, lines: List): Boolean { + var budget = MAX_BODY_BYTES + val kept = ArrayList(lines.size) + for (i in lines.indices.reversed()) { + // +8 ≈ JSON quotes/comma + escaping slack per line. + val cost = lines[i].toByteArray(Charsets.UTF_8).size + 8 + if (cost > budget) break + budget -= cost + kept.add(lines[i]) + } + kept.reverse() + val envelope = Envelope( + deviceId = deviceId(context), + appVersion = "${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})", + sentAt = isoNowUtc(), + reason = reason, + lines = kept, + ) + val body = Json.encodeToString(envelope) + .toRequestBody("application/json; charset=utf-8".toMediaType()) + val request = Request.Builder() + .url(INGEST_URL) + .header("Authorization", "Bearer ${BuildConfig.STRAW_LOGS_TOKEN}") + .post(body) + .build() + return http.newCall(request).execute().use { it.isSuccessful } + } +} diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt index 6c96d72d0..35dee1e58 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt @@ -66,6 +66,7 @@ import com.sulkta.straw.data.Settings import com.sulkta.straw.data.ThemeMode import com.sulkta.straw.feature.dataimport.ImportResult import com.sulkta.straw.feature.dataimport.SettingsImport +import com.sulkta.straw.feature.diag.LogShipper import com.sulkta.straw.feature.feed.SubscriptionFeedViewModel import com.sulkta.straw.feature.search.SearchViewModel import com.sulkta.straw.util.LogDump @@ -774,6 +775,39 @@ fun SettingsScreen() { Text(if (logDumping) "Exporting…" else "Export logs…") } + // Dogfood log shipping — only rendered in builds with an + // injected ingest token (LogShipper.enabled is compile-time + // false otherwise, so contributor builds show no phantom row). + if (LogShipper.enabled) { + Spacer(modifier = Modifier.height(16.dp)) + Text( + "Or send the same scrubbed log dump straight to Kayos " + + "(logs.sulkta.com) — no share sheet. URLs, video ids, " + + "and anything token-shaped are redacted on-device first.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Spacer(modifier = Modifier.height(12.dp)) + var logSending by remember { mutableStateOf(false) } + OutlinedButton( + enabled = !logSending, + onClick = { + logSending = true + scope.launch { + val ok = LogShipper.send(context, reason = "manual") + logSending = false + Toast.makeText( + context, + if (ok) "Logs sent — thanks!" else "Couldn't reach the log server", + Toast.LENGTH_LONG, + ).show() + } + }, + ) { + Text(if (logSending) "Sending…" else "Send logs to Kayos") + } + } + Spacer(modifier = Modifier.height(32.dp)) Text( "Import from NewPipe / Tubular", diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt b/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt index 159e7b8aa..ec922c4bd 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt @@ -96,10 +96,46 @@ object LogDump { } /** - * Pre-redact known credential-shaped substrings before they hit - * disk. Cheap line-level pass — adversarial-perfect would need a - * URL parser, but the regex approach catches every documented - * leak vector at zero allocation cost. + * Pull recent logcat, scrub every line, keep only the newest + * [maxLines]. BLOCKING — callers must already be off the main + * thread (LogShipper runs on Dispatchers.IO; the crash handler + * runs this on its own short-lived worker thread, where a suspend + * fun can't be awaited without an event loop). + * + * Best-effort by design: a logcat exec failure returns whatever + * was read (possibly empty) instead of throwing, so the crash + * shipper can still deliver the stacktrace. + */ + fun captureScrubbedLinesBlocking(maxLines: Int): List { + val ring = ArrayDeque() + runCatching { + val pid = Process.myPid() + val cmd = arrayOf("logcat", "-d", "-v", "threadtime", "--pid=$pid") + val proc = ProcessBuilder(*cmd).redirectErrorStream(true).start() + proc.inputStream.bufferedReader().useLines { lines -> + lines.forEach { line -> + if (ring.size == maxLines) ring.removeFirst() + ring.addLast(line) + } + } + proc.waitFor() + } + // Scrub only the retained window — cheaper than scrubbing lines + // that get dropped, and nothing raw ever leaves this function. + return ring.map(::scrubLine) + } + + /** + * Pre-redact known credential- and identity-shaped substrings + * before they hit disk or the wire. Cheap line-level pass — + * adversarial-perfect would need a URL parser, but the regex + * approach catches every documented leak vector at zero + * allocation cost. + * + * BIAS: over-redaction. These lines leave the device (share sheet + * + LogShipper → logs.sulkta.com), so a false positive costs a + * little debug signal while a false negative leaks what someone + * watched. Anything URL-, token-, id-, or address-shaped goes. * * Public so error-handler call sites (PlayerScreen / VideoDetail * `playbackError`) can scrub Media3's `PlaybackException.message` @@ -109,28 +145,112 @@ object LogDump { */ fun scrubLine(line: String): String { var s = line - // Pre-signed googlevideo URLs: keep host visible, drop path+query. + // Pre-signed googlevideo URLs: keep the host label visible, drop + // host prefix + path + query (session-bound streaming creds). s = GOOGLEVIDEO_URL_RE.replace(s, "https://.googlevideo.com/") + // Credential-shaped headers / key-value pairs, wherever they + // appear: `Authorization: Bearer x`, `cookie: …`, `api_key=…`. + s = BEARER_RE.replace(s, "$1 ") + s = CRED_KV_RE.replace(s, "$1: ") // Long, distinctive token names — match anywhere. s = SIGNED_PARAM_LONG_RE.replace(s, "$1=") // Short single-letter / two-letter tokens — require `[?&]` // immediately before to avoid eating innocent counters. s = SIGNED_PARAM_SHORT_RE.replace(s, "$1$2=") + // ANY remaining URL: keep scheme+host (routing/debug signal, + // not PII), drop path + query — that's where watch?v=…, + // /vi// thumbnails, search terms, etc. live. + s = URL_TAIL_RE.replace(s, "$1/") + // Schemeless YT links pasted into log messages ("youtu.be/xyz"). + s = SCHEMELESS_YT_RE.replace(s, "$1/") + // Emails. + s = EMAIL_RE.replace(s, "") + // YouTube channel + playlist ids (watch behavior). + s = CHANNEL_ID_RE.replace(s, "") + s = PLAYLIST_ID_RE.replace(s, "") + // Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged + // outside any URL (rustypipe/strawcore do this). Requiring a + // digit/_/- inside the run spares ordinary 11-letter words; + // real ids virtually always contain one. Pure-alpha ids are + // still caught earlier when keyed (v=…) or inside a URL. + s = VIDEO_ID_CANDIDATE_RE.replace(s) { m -> + if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "" else m.value + } + // Long high-entropy runs (hashes, visitor data, unlabeled + // tokens): 20+ [A-Za-z0-9_-] chars containing a digit. + s = LONG_TOKEN_RE.replace(s, "") + // IP addresses (v4 + v6 — v6 patterns are shaped so threadtime + // HH:MM:SS timestamps can never match). + s = IPV4_RE.replace(s, "") + s = IPV6_FULL_RE.replace(s, "") + s = IPV6_COMPRESSED_RE.replace(s, "") return s } private val GOOGLEVIDEO_URL_RE = Regex( """https?://[a-zA-Z0-9.-]*googlevideo\.com/\S+""", ) + // `Authorization: Bearer ` and friends. The token charset is + // the RFC 6750 b64token alphabet. + private val BEARER_RE = Regex( + """\b(bearer)\s+[A-Za-z0-9._~+/=-]{4,}""", + RegexOption.IGNORE_CASE, + ) + // Credential-y names followed by `: value` or `= value` (optional + // closing quote for JSON `"token":"…"` shapes). Values may be + // quoted strings or a bare token. + private val CRED_KV_RE = Regex( + """\b(authorization|proxy-authorization|cookie|set-cookie|x-goog-[a-z\-]+|x-youtube-[a-z\-]+|api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|client[-_]?secret|secret|token|auth|session[-_]?id|passw(?:or)?d|pwd|mnemonic|private[-_]?key)"?\s*[:=]\s*("[^"]*"|\S+)""", + RegexOption.IGNORE_CASE, + ) // Long tokens are unique enough to match anywhere. Short tokens - // (n, mn, ms, mo, pl, ip, ei) require `[?&]` immediately before - // so we don't redact innocuous `n=42` counters from other libs. + // (n, mn, ms, … v, id, q) require `[?&]` immediately before so we + // don't redact innocuous `n=42` counters from other libs. `v`, + // `id`, `list`, `q`, `t` are the YouTube identity/search params. private val SIGNED_PARAM_LONG_RE = Regex( - """\b(signature|sparams|lsig|cpn|expire|pot|sig|key)=([^&\s"']+)""", + """\b(signature|sparams|lsig|cpn|expire|pot|sig|key|videoId|video_id|docid|search_query|query)=([^&\s"']+)""", RegexOption.IGNORE_CASE, ) private val SIGNED_PARAM_SHORT_RE = Regex( - """([?&])(n|mn|ms|mo|pl|ip|ei)=([^&\s"']+)""", + """([?&])(n|mn|ms|mo|pl|ip|ei|v|id|list|q|t)=([^&\s"']+)""", RegexOption.IGNORE_CASE, ) + // Any http(s) URL: capture scheme + host(+port), scrub the rest. + // Host charset can't contain `<`, so the googlevideo replacement + // above is never re-matched. + private val URL_TAIL_RE = Regex( + """(https?://[A-Za-z0-9.-]+(?::\d+)?)[/?#][^\s"'<>]*""", + ) + // youtu.be/… + youtube.com/… without a scheme. The lookbehind + // skips hosts already handled as part of a full URL (preceded by + // `/` or a subdomain dot). + private val SCHEMELESS_YT_RE = Regex( + """(?]*""", + ) + private val EMAIL_RE = Regex( + """[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}""", + ) + private val CHANNEL_ID_RE = Regex( + """(?