feat(diag): dogfood log-shipping to logs.sulkta.com
Ship scrubbed device logs to our self-hosted ingest (logs.sulkta.com) so a 'green in every offline gate, broken on-device' regression (the visionOS playback outage) can be caught end-to-end. LogShipper POSTs a scrubbed log ring + crash stacktrace on (a) an uncaught exception (chained to the previous handler) and (b) a 'Send logs to Kayos' Settings button. Everything runs through LogDump.scrubLine first (URLs/video-ids/emails/tokens redacted); the server re-scrubs. device_id = a random per-install UUID (no hardware id). Compile-DISABLED unless the STRAW_LOGS_INGEST_TOKEN build secret is set, so contributor forks never phone home. build.yml passes the secret to the debug build.
This commit is contained in:
parent
5021749975
commit
b5ef1e1973
6 changed files with 429 additions and 9 deletions
|
|
@ -88,6 +88,10 @@ jobs:
|
|||
STRAW_KEYSTORE_PASS: android
|
||||
STRAW_KEY_ALIAS: androiddebugkey
|
||||
STRAW_KEY_PASS: android
|
||||
# Dogfood log-shipping: the app is compile-DISABLED unless this is set
|
||||
# (empty ⇒ no crash handler, no network, Settings row hidden). Repo
|
||||
# secret; absent on contributor forks so their builds don't phone home.
|
||||
STRAW_LOGS_INGEST_TOKEN: ${{ secrets.STRAW_LOGS_INGEST_TOKEN }}
|
||||
# Keep the 4-ABI cross-compile off the container rootfs.
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/cargo-target
|
||||
# ionice idle class + low nice so the build yields disk/CPU to the
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
*/
|
||||
|
||||
import com.android.build.api.dsl.ApplicationExtension
|
||||
import java.util.Properties
|
||||
|
||||
plugins {
|
||||
alias(libs.plugins.android.application)
|
||||
|
|
@ -36,6 +37,30 @@ configure<ApplicationExtension> {
|
|||
versionCode = STRAW_VERSION_CODE
|
||||
versionName = STRAW_VERSION_NAME
|
||||
resValue("string", "app_name", "Straw")
|
||||
|
||||
// Dogfood log-shipping token (feature/diag/LogShipper.kt). This repo
|
||||
// is PUBLIC — the token must NEVER be committed, so it's injected at
|
||||
// build time and defaults to "":
|
||||
// 1. env var STRAW_LOGS_INGEST_TOKEN — CI: add a Forgejo Actions
|
||||
// repo secret STRAW_LOGS_INGEST_TOKEN and pass it in the
|
||||
// "Assemble debug APK" step's env block in
|
||||
// .forgejo/workflows/build.yml.
|
||||
// 2. `straw.logs.ingest.token=…` in local.properties (gitignored)
|
||||
// for local dogfood builds.
|
||||
// Empty token ⇒ LogShipper is fully disabled (no-op, zero network,
|
||||
// no Settings row) — contributor builds neither break nor phone home.
|
||||
val strawLogsToken: String = System.getenv("STRAW_LOGS_INGEST_TOKEN")
|
||||
?: run {
|
||||
val props = Properties()
|
||||
val f = rootProject.file("local.properties")
|
||||
if (f.exists()) f.inputStream().use { props.load(it) }
|
||||
props.getProperty("straw.logs.ingest.token")
|
||||
}
|
||||
?: ""
|
||||
// Escape so a pathological token can't break out of the generated
|
||||
// String literal in BuildConfig.java.
|
||||
val escapedToken = strawLogsToken.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
buildConfigField("String", "STRAW_LOGS_TOKEN", "\"$escapedToken\"")
|
||||
}
|
||||
|
||||
// Explicit signing so CI / release builds reuse ONE keystore instead of
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import com.sulkta.straw.data.SearchCache
|
|||
import com.sulkta.straw.data.Settings
|
||||
import com.sulkta.straw.data.Subscriptions
|
||||
import com.sulkta.straw.feature.dataimport.SettingsImport
|
||||
import com.sulkta.straw.feature.diag.LogShipper
|
||||
import com.sulkta.straw.feature.feed.FeedRefreshScheduler
|
||||
import com.sulkta.straw.feature.update.UpdateScheduler
|
||||
import com.sulkta.straw.feature.update.runUpdateCheck
|
||||
|
|
@ -90,6 +91,12 @@ class StrawApp : Application(), SingletonImageLoader.Factory {
|
|||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
// Dogfood crash shipping — wrap the default uncaught-exception
|
||||
// handler FIRST so even an init crash below gets shipped. The
|
||||
// wrapper always chains to the previous handler (normal crash
|
||||
// behavior is untouched) and is a no-op in builds without an
|
||||
// injected STRAW_LOGS_TOKEN.
|
||||
LogShipper.installCrashHandler(this)
|
||||
// Path C-7: route Rust `log::*` calls into Android logcat under tag
|
||||
// "strawcore". Without this, every log line emitted from rustypipe /
|
||||
// strawcore is silently dropped, making playback regressions invisible
|
||||
|
|
|
|||
|
|
@ -0,0 +1,230 @@
|
|||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Sulkta
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
*
|
||||
* Dogfood log shipping — POST a scrubbed logcat window to Kayos's
|
||||
* ingest service (logs.sulkta.com). Two triggers:
|
||||
*
|
||||
* * crash — a default-UncaughtExceptionHandler wrapper captures the
|
||||
* log ring + the throwable's stacktrace, attempts one
|
||||
* hard-capped best-effort send, then chains to the
|
||||
* previously-installed handler so the process still dies
|
||||
* (and Android still shows its crash dialog) normally.
|
||||
* * manual — the Settings → Diagnostics "Send logs to Kayos" button.
|
||||
*
|
||||
* PRIVACY: every line — including the crash stacktrace — goes through
|
||||
* LogDump.scrubLine before leaving the device; the server re-scrubs,
|
||||
* but the phone is the primary gate. device_id is a per-install random
|
||||
* UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole
|
||||
* feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was
|
||||
* injected (see strawApp/build.gradle.kts) — for everyone else
|
||||
* `enabled` is a compile-time false, so R8 strips the network path and
|
||||
* a contributor build cannot phone home.
|
||||
*
|
||||
* DELIVERY: strictly best-effort. Any failure (offline, 401/413/429,
|
||||
* timeout) is swallowed; nothing here may ever crash or block the app.
|
||||
*/
|
||||
|
||||
package com.sulkta.straw.feature.diag
|
||||
|
||||
import android.content.Context
|
||||
import com.sulkta.straw.BuildConfig
|
||||
import com.sulkta.straw.util.LogDump
|
||||
import com.sulkta.straw.util.runCatchingCancellable
|
||||
import java.io.PrintWriter
|
||||
import java.io.StringWriter
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.Date
|
||||
import java.util.Locale
|
||||
import java.util.TimeZone
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
|
||||
object LogShipper {
|
||||
|
||||
/** Not a secret — only the bearer token is. */
|
||||
private const val INGEST_URL = "https://logs.sulkta.com/ingest"
|
||||
|
||||
/** Server caps: 10k lines / 1 MB body. Stay well under both. */
|
||||
private const val MAX_LINES = 2_000
|
||||
private const val MAX_BODY_BYTES = 700_000
|
||||
|
||||
/** Hard cap on how long the crash path may delay process death. */
|
||||
private const val CRASH_JOIN_MS = 4_000L
|
||||
|
||||
private const val PREFS = "straw_diag"
|
||||
private const val KEY_DEVICE_ID = "device_id"
|
||||
|
||||
/**
|
||||
* Compile-time gate: builds without an injected token have no
|
||||
* shipping at all — no network, no crash handler, no Settings row.
|
||||
*/
|
||||
val enabled: Boolean
|
||||
get() = BuildConfig.STRAW_LOGS_TOKEN.isNotEmpty()
|
||||
|
||||
// Short timeouts — this is telemetry, not payload. callTimeout is
|
||||
// the absolute ceiling on the whole request so nothing here can
|
||||
// hold a WorkManager-style wakelock pattern open.
|
||||
private val http: OkHttpClient by lazy {
|
||||
OkHttpClient.Builder()
|
||||
.connectTimeout(5, TimeUnit.SECONDS)
|
||||
.writeTimeout(5, TimeUnit.SECONDS)
|
||||
.readTimeout(5, TimeUnit.SECONDS)
|
||||
.callTimeout(10, TimeUnit.SECONDS)
|
||||
.build()
|
||||
}
|
||||
|
||||
@Serializable
|
||||
private data class Envelope(
|
||||
@SerialName("device_id") val deviceId: String,
|
||||
@SerialName("app_version") val appVersion: String,
|
||||
@SerialName("sent_at") val sentAt: String,
|
||||
val reason: String,
|
||||
val lines: List<String>,
|
||||
)
|
||||
|
||||
/**
|
||||
* Stable per-install id: `straw-<random uuid>`, minted once and
|
||||
* persisted. Deliberately NOT ANDROID_ID/IMEI — it identifies an
|
||||
* install for log correlation, never the hardware or the person.
|
||||
* commit() (not apply()) because both call paths are already on a
|
||||
* background thread and the crash path dies right after — an
|
||||
* async write would lose the id and mint a new one per crash.
|
||||
*/
|
||||
private fun deviceId(context: Context): String {
|
||||
val sp = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE)
|
||||
sp.getString(KEY_DEVICE_ID, null)?.let { return it }
|
||||
val id = "straw-${UUID.randomUUID()}"
|
||||
sp.edit().putString(KEY_DEVICE_ID, id).commit()
|
||||
return id
|
||||
}
|
||||
|
||||
private fun isoNowUtc(): String =
|
||||
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US)
|
||||
.apply { timeZone = TimeZone.getTimeZone("UTC") }
|
||||
.format(Date())
|
||||
|
||||
/**
|
||||
* Manual trigger (Settings). Captures + scrubs the log ring and
|
||||
* POSTs it. Returns true iff the server 2xx'd — the UI shows a
|
||||
* toast either way; failures never throw.
|
||||
*/
|
||||
suspend fun send(context: Context, reason: String): Boolean =
|
||||
withContext(Dispatchers.IO) {
|
||||
if (!enabled) return@withContext false
|
||||
runCatchingCancellable {
|
||||
postLines(
|
||||
context = context,
|
||||
reason = reason,
|
||||
lines = LogDump.captureScrubbedLinesBlocking(MAX_LINES),
|
||||
)
|
||||
}.getOrDefault(false)
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap the process-default uncaught-exception handler. Ship-then-
|
||||
* chain: whatever happens in our path (including a throw), the
|
||||
* previous handler ALWAYS runs, so the app still crashes/reports
|
||||
* exactly as before. No-op without a token.
|
||||
*/
|
||||
fun installCrashHandler(context: Context) {
|
||||
if (!enabled) return
|
||||
val appContext = context.applicationContext
|
||||
val previous = Thread.getDefaultUncaughtExceptionHandler()
|
||||
Thread.setDefaultUncaughtExceptionHandler { thread, throwable ->
|
||||
try {
|
||||
shipCrash(appContext, throwable)
|
||||
} catch (_: Throwable) {
|
||||
// Never let diagnostics mask the real crash.
|
||||
}
|
||||
previous?.uncaughtException(thread, throwable)
|
||||
}
|
||||
}
|
||||
|
||||
@Volatile
|
||||
private var crashShipStarted = false
|
||||
|
||||
/**
|
||||
* Crash-path send. The crashing thread may be main (network there
|
||||
* throws NetworkOnMainThreadException) and is about to die either
|
||||
* way, so the capture+POST runs on a dedicated worker thread and
|
||||
* we join with a hard cap — the death is delayed at most
|
||||
* [CRASH_JOIN_MS], never hung.
|
||||
*/
|
||||
private fun shipCrash(context: Context, throwable: Throwable) {
|
||||
// One attempt per process — also breaks recursion if the ship
|
||||
// path itself crashes on another thread mid-flight.
|
||||
if (crashShipStarted) return
|
||||
crashShipStarted = true
|
||||
val worker = Thread {
|
||||
try {
|
||||
val trace = StringWriter().also {
|
||||
throwable.printStackTrace(PrintWriter(it))
|
||||
}.toString()
|
||||
.split('\n')
|
||||
.take(400)
|
||||
.map(LogDump::scrubLine)
|
||||
val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES)
|
||||
postLines(
|
||||
context = context,
|
||||
reason = "crash",
|
||||
lines = ring + "--- uncaught exception ---" + trace,
|
||||
)
|
||||
} catch (_: Throwable) {
|
||||
// Best-effort only.
|
||||
}
|
||||
}
|
||||
worker.name = "straw-crash-ship"
|
||||
worker.isDaemon = true
|
||||
worker.start()
|
||||
try {
|
||||
worker.join(CRASH_JOIN_MS)
|
||||
} catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the envelope and POST. `lines` MUST already be scrubbed
|
||||
* (both call paths run everything through LogDump.scrubLine).
|
||||
* Trims oldest-first to stay under the server's 1 MB cap — one
|
||||
* bounded request, no chunking needed at our line cap.
|
||||
*/
|
||||
private fun postLines(context: Context, reason: String, lines: List<String>): Boolean {
|
||||
var budget = MAX_BODY_BYTES
|
||||
val kept = ArrayList<String>(lines.size)
|
||||
for (i in lines.indices.reversed()) {
|
||||
// +8 ≈ JSON quotes/comma + escaping slack per line.
|
||||
val cost = lines[i].toByteArray(Charsets.UTF_8).size + 8
|
||||
if (cost > budget) break
|
||||
budget -= cost
|
||||
kept.add(lines[i])
|
||||
}
|
||||
kept.reverse()
|
||||
val envelope = Envelope(
|
||||
deviceId = deviceId(context),
|
||||
appVersion = "${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})",
|
||||
sentAt = isoNowUtc(),
|
||||
reason = reason,
|
||||
lines = kept,
|
||||
)
|
||||
val body = Json.encodeToString(envelope)
|
||||
.toRequestBody("application/json; charset=utf-8".toMediaType())
|
||||
val request = Request.Builder()
|
||||
.url(INGEST_URL)
|
||||
.header("Authorization", "Bearer ${BuildConfig.STRAW_LOGS_TOKEN}")
|
||||
.post(body)
|
||||
.build()
|
||||
return http.newCall(request).execute().use { it.isSuccessful }
|
||||
}
|
||||
}
|
||||
|
|
@ -66,6 +66,7 @@ import com.sulkta.straw.data.Settings
|
|||
import com.sulkta.straw.data.ThemeMode
|
||||
import com.sulkta.straw.feature.dataimport.ImportResult
|
||||
import com.sulkta.straw.feature.dataimport.SettingsImport
|
||||
import com.sulkta.straw.feature.diag.LogShipper
|
||||
import com.sulkta.straw.feature.feed.SubscriptionFeedViewModel
|
||||
import com.sulkta.straw.feature.search.SearchViewModel
|
||||
import com.sulkta.straw.util.LogDump
|
||||
|
|
@ -774,6 +775,39 @@ fun SettingsScreen() {
|
|||
Text(if (logDumping) "Exporting…" else "Export logs…")
|
||||
}
|
||||
|
||||
// Dogfood log shipping — only rendered in builds with an
|
||||
// injected ingest token (LogShipper.enabled is compile-time
|
||||
// false otherwise, so contributor builds show no phantom row).
|
||||
if (LogShipper.enabled) {
|
||||
Spacer(modifier = Modifier.height(16.dp))
|
||||
Text(
|
||||
"Or send the same scrubbed log dump straight to Kayos " +
|
||||
"(logs.sulkta.com) — no share sheet. URLs, video ids, " +
|
||||
"and anything token-shaped are redacted on-device first.",
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
Spacer(modifier = Modifier.height(12.dp))
|
||||
var logSending by remember { mutableStateOf(false) }
|
||||
OutlinedButton(
|
||||
enabled = !logSending,
|
||||
onClick = {
|
||||
logSending = true
|
||||
scope.launch {
|
||||
val ok = LogShipper.send(context, reason = "manual")
|
||||
logSending = false
|
||||
Toast.makeText(
|
||||
context,
|
||||
if (ok) "Logs sent — thanks!" else "Couldn't reach the log server",
|
||||
Toast.LENGTH_LONG,
|
||||
).show()
|
||||
}
|
||||
},
|
||||
) {
|
||||
Text(if (logSending) "Sending…" else "Send logs to Kayos")
|
||||
}
|
||||
}
|
||||
|
||||
Spacer(modifier = Modifier.height(32.dp))
|
||||
Text(
|
||||
"Import from NewPipe / Tubular",
|
||||
|
|
|
|||
|
|
@ -96,10 +96,46 @@ object LogDump {
|
|||
}
|
||||
|
||||
/**
|
||||
* Pre-redact known credential-shaped substrings before they hit
|
||||
* disk. Cheap line-level pass — adversarial-perfect would need a
|
||||
* URL parser, but the regex approach catches every documented
|
||||
* leak vector at zero allocation cost.
|
||||
* Pull recent logcat, scrub every line, keep only the newest
|
||||
* [maxLines]. BLOCKING — callers must already be off the main
|
||||
* thread (LogShipper runs on Dispatchers.IO; the crash handler
|
||||
* runs this on its own short-lived worker thread, where a suspend
|
||||
* fun can't be awaited without an event loop).
|
||||
*
|
||||
* Best-effort by design: a logcat exec failure returns whatever
|
||||
* was read (possibly empty) instead of throwing, so the crash
|
||||
* shipper can still deliver the stacktrace.
|
||||
*/
|
||||
fun captureScrubbedLinesBlocking(maxLines: Int): List<String> {
|
||||
val ring = ArrayDeque<String>()
|
||||
runCatching {
|
||||
val pid = Process.myPid()
|
||||
val cmd = arrayOf("logcat", "-d", "-v", "threadtime", "--pid=$pid")
|
||||
val proc = ProcessBuilder(*cmd).redirectErrorStream(true).start()
|
||||
proc.inputStream.bufferedReader().useLines { lines ->
|
||||
lines.forEach { line ->
|
||||
if (ring.size == maxLines) ring.removeFirst()
|
||||
ring.addLast(line)
|
||||
}
|
||||
}
|
||||
proc.waitFor()
|
||||
}
|
||||
// Scrub only the retained window — cheaper than scrubbing lines
|
||||
// that get dropped, and nothing raw ever leaves this function.
|
||||
return ring.map(::scrubLine)
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-redact known credential- and identity-shaped substrings
|
||||
* before they hit disk or the wire. Cheap line-level pass —
|
||||
* adversarial-perfect would need a URL parser, but the regex
|
||||
* approach catches every documented leak vector at zero
|
||||
* allocation cost.
|
||||
*
|
||||
* BIAS: over-redaction. These lines leave the device (share sheet
|
||||
* + LogShipper → logs.sulkta.com), so a false positive costs a
|
||||
* little debug signal while a false negative leaks what someone
|
||||
* watched. Anything URL-, token-, id-, or address-shaped goes.
|
||||
*
|
||||
* Public so error-handler call sites (PlayerScreen / VideoDetail
|
||||
* `playbackError`) can scrub Media3's `PlaybackException.message`
|
||||
|
|
@ -109,28 +145,112 @@ object LogDump {
|
|||
*/
|
||||
fun scrubLine(line: String): String {
|
||||
var s = line
|
||||
// Pre-signed googlevideo URLs: keep host visible, drop path+query.
|
||||
// Pre-signed googlevideo URLs: keep the host label visible, drop
|
||||
// host prefix + path + query (session-bound streaming creds).
|
||||
s = GOOGLEVIDEO_URL_RE.replace(s, "https://<host>.googlevideo.com/<scrubbed>")
|
||||
// Credential-shaped headers / key-value pairs, wherever they
|
||||
// appear: `Authorization: Bearer x`, `cookie: …`, `api_key=…`.
|
||||
s = BEARER_RE.replace(s, "$1 <scrubbed>")
|
||||
s = CRED_KV_RE.replace(s, "$1: <scrubbed>")
|
||||
// Long, distinctive token names — match anywhere.
|
||||
s = SIGNED_PARAM_LONG_RE.replace(s, "$1=<scrubbed>")
|
||||
// Short single-letter / two-letter tokens — require `[?&]`
|
||||
// immediately before to avoid eating innocent counters.
|
||||
s = SIGNED_PARAM_SHORT_RE.replace(s, "$1$2=<scrubbed>")
|
||||
// ANY remaining URL: keep scheme+host (routing/debug signal,
|
||||
// not PII), drop path + query — that's where watch?v=…,
|
||||
// /vi/<id>/ thumbnails, search terms, etc. live.
|
||||
s = URL_TAIL_RE.replace(s, "$1/<scrubbed>")
|
||||
// Schemeless YT links pasted into log messages ("youtu.be/xyz").
|
||||
s = SCHEMELESS_YT_RE.replace(s, "$1/<scrubbed>")
|
||||
// Emails.
|
||||
s = EMAIL_RE.replace(s, "<email>")
|
||||
// YouTube channel + playlist ids (watch behavior).
|
||||
s = CHANNEL_ID_RE.replace(s, "<channelId>")
|
||||
s = PLAYLIST_ID_RE.replace(s, "<playlistId>")
|
||||
// Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged
|
||||
// outside any URL (rustypipe/strawcore do this). Requiring a
|
||||
// digit/_/- inside the run spares ordinary 11-letter words;
|
||||
// real ids virtually always contain one. Pure-alpha ids are
|
||||
// still caught earlier when keyed (v=…) or inside a URL.
|
||||
s = VIDEO_ID_CANDIDATE_RE.replace(s) { m ->
|
||||
if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "<videoId>" else m.value
|
||||
}
|
||||
// Long high-entropy runs (hashes, visitor data, unlabeled
|
||||
// tokens): 20+ [A-Za-z0-9_-] chars containing a digit.
|
||||
s = LONG_TOKEN_RE.replace(s, "<token>")
|
||||
// IP addresses (v4 + v6 — v6 patterns are shaped so threadtime
|
||||
// HH:MM:SS timestamps can never match).
|
||||
s = IPV4_RE.replace(s, "<ip>")
|
||||
s = IPV6_FULL_RE.replace(s, "<ip>")
|
||||
s = IPV6_COMPRESSED_RE.replace(s, "<ip>")
|
||||
return s
|
||||
}
|
||||
|
||||
private val GOOGLEVIDEO_URL_RE = Regex(
|
||||
"""https?://[a-zA-Z0-9.-]*googlevideo\.com/\S+""",
|
||||
)
|
||||
// `Authorization: Bearer <jwt>` and friends. The token charset is
|
||||
// the RFC 6750 b64token alphabet.
|
||||
private val BEARER_RE = Regex(
|
||||
"""\b(bearer)\s+[A-Za-z0-9._~+/=-]{4,}""",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
// Credential-y names followed by `: value` or `= value` (optional
|
||||
// closing quote for JSON `"token":"…"` shapes). Values may be
|
||||
// quoted strings or a bare token.
|
||||
private val CRED_KV_RE = Regex(
|
||||
"""\b(authorization|proxy-authorization|cookie|set-cookie|x-goog-[a-z\-]+|x-youtube-[a-z\-]+|api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|client[-_]?secret|secret|token|auth|session[-_]?id|passw(?:or)?d|pwd|mnemonic|private[-_]?key)"?\s*[:=]\s*("[^"]*"|\S+)""",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
// Long tokens are unique enough to match anywhere. Short tokens
|
||||
// (n, mn, ms, mo, pl, ip, ei) require `[?&]` immediately before
|
||||
// so we don't redact innocuous `n=42` counters from other libs.
|
||||
// (n, mn, ms, … v, id, q) require `[?&]` immediately before so we
|
||||
// don't redact innocuous `n=42` counters from other libs. `v`,
|
||||
// `id`, `list`, `q`, `t` are the YouTube identity/search params.
|
||||
private val SIGNED_PARAM_LONG_RE = Regex(
|
||||
"""\b(signature|sparams|lsig|cpn|expire|pot|sig|key)=([^&\s"']+)""",
|
||||
"""\b(signature|sparams|lsig|cpn|expire|pot|sig|key|videoId|video_id|docid|search_query|query)=([^&\s"']+)""",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
private val SIGNED_PARAM_SHORT_RE = Regex(
|
||||
"""([?&])(n|mn|ms|mo|pl|ip|ei)=([^&\s"']+)""",
|
||||
"""([?&])(n|mn|ms|mo|pl|ip|ei|v|id|list|q|t)=([^&\s"']+)""",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
// Any http(s) URL: capture scheme + host(+port), scrub the rest.
|
||||
// Host charset can't contain `<`, so the googlevideo replacement
|
||||
// above is never re-matched.
|
||||
private val URL_TAIL_RE = Regex(
|
||||
"""(https?://[A-Za-z0-9.-]+(?::\d+)?)[/?#][^\s"'<>]*""",
|
||||
)
|
||||
// youtu.be/… + youtube.com/… without a scheme. The lookbehind
|
||||
// skips hosts already handled as part of a full URL (preceded by
|
||||
// `/` or a subdomain dot).
|
||||
private val SCHEMELESS_YT_RE = Regex(
|
||||
"""(?<![/@.\w])((?:www\.|m\.|music\.)?youtube\.com|youtu\.be)[/?][^\s"'<>]*""",
|
||||
)
|
||||
private val EMAIL_RE = Regex(
|
||||
"""[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}""",
|
||||
)
|
||||
private val CHANNEL_ID_RE = Regex(
|
||||
"""(?<![A-Za-z0-9_-])UC[A-Za-z0-9_-]{22}(?![A-Za-z0-9_-])""",
|
||||
)
|
||||
private val PLAYLIST_ID_RE = Regex(
|
||||
"""(?<![A-Za-z0-9_-])(?:PL|UU|LL|RD|OLAK5uy_)[A-Za-z0-9_-]{10,}(?![A-Za-z0-9_-])""",
|
||||
)
|
||||
private val VIDEO_ID_CANDIDATE_RE = Regex(
|
||||
"""(?<![A-Za-z0-9_-])[A-Za-z0-9_-]{11}(?![A-Za-z0-9_-])""",
|
||||
)
|
||||
private val LONG_TOKEN_RE = Regex(
|
||||
"""(?<![A-Za-z0-9_-])(?=[A-Za-z_-]*\d)[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])""",
|
||||
)
|
||||
private val IPV4_RE = Regex(
|
||||
"""(?<!\d)(?:\d{1,3}\.){3}\d{1,3}(?!\d)""",
|
||||
)
|
||||
// Full-form v6 needs 6-8 hex groups — `HH:MM:SS` (3 groups) can't
|
||||
// match. Compressed form requires a literal `::`.
|
||||
private val IPV6_FULL_RE = Regex(
|
||||
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){5,7}[0-9A-Fa-f]{1,4}(?![0-9A-Fa-f:])""",
|
||||
)
|
||||
private val IPV6_COMPRESSED_RE = Regex(
|
||||
"""(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{1,4}:){1,6}:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4}){0,5})?(?![0-9A-Fa-f:])""",
|
||||
)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue