diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 000000000..7d2273bb5 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,167 @@ +# Straw APK — build on Forgejo Actions, and (gated on a green build) publish to +# fdroid.sulkta.com so the in-app updater picks it up. +# +# Runs in the dedicated git.sulkta.com/sulkta-infra/straw-build image (Android +# SDK/NDK + Rust + cargo-ndk, see ci/Dockerfile). The signing key comes from +# the STRAW_SIGNING_KEYSTORE_B64 secret (the same androiddebugkey the whole +# series is signed with — vaulted as "Sulkta — Straw fdroid signing keystore"). +# +# Publish is ROOTLESS (#444): the signed APK is handed to a forced-command +# script on the Lucy host (key STRAW_FDROID_LUCY_KEY, pinned to that one script +# via restrict,command=). The host does the fdroid index re-sign + the Rackham +# rsync — the signing keystore never enters this CI container, and this job no +# longer needs the host docker socket. The host script re-verifies the APK +# signer before publishing. +name: build-apk + +on: + push: + branches: [main] + paths: + - 'strawApp/**' + - 'rust/**' + - 'buildSrc/**' + - 'gradle/**' + - 'gradlew' + - '*.gradle.kts' + - 'gradle.properties' + - 'ci/Dockerfile' + - '.forgejo/workflows/build.yml' + workflow_dispatch: + +jobs: + build-and-publish: + runs-on: lucy-rust + # Run every step with bash, not the runner's default dash — the steps use + # `set -euo pipefail` (pipefail is a bashism; dash errors "Illegal option"). + # The straw-build image ships /usr/bin/bash. + defaults: + run: + shell: bash + container: + image: git.sulkta.com/sulkta-infra/straw-build:latest + steps: + # We clone with plain git instead of actions/checkout@v4: that action is + # a Node action, and the straw-build job container ships the Android + + # Rust toolchain but NOT node — so checkout@v4 dies with + # `exec: "node": not found`. git is in the image, both repos are public, + # and a shell clone also sidesteps the runner's flaky data.forgejo.org + # action fetch. strawcore must be a SIBLING of straw because + # rust/strawcore depends on it via `path = "../../../strawcore"`. + - name: Checkout straw + strawcore (sibling, no JS actions) + run: | + set -euo pipefail + git clone https://git.sulkta.com/Sulkta-OSS/straw.git straw + git -C straw checkout --detach "${{ github.sha }}" + git clone --depth 1 https://git.sulkta.com/Sulkta-OSS/strawcore.git strawcore + echo "straw: $(git -C straw rev-parse --short HEAD)" + echo "strawcore: $(git -C strawcore rev-parse --short HEAD)" + + - name: Decode signing keystore + env: + KS_B64: ${{ secrets.STRAW_SIGNING_KEYSTORE_B64 }} + run: echo "$KS_B64" | base64 -d > "$GITHUB_WORKSPACE/straw.keystore" + + # Auto-derive versionCode from the main commit count so a release can + # NEVER silently reuse a code — the 2026-07-29 trap where a commit that + # didn't hand-bump ProjectConfig rebuilt the same debug_.apk, the + # publish receiver's anti-downgrade guard refused the duplicate (exit 3 = + # "nothing to do"), and the CI went green having shipped nothing. The + # OFFSET keeps the number continuous with the manual era (last hand-set + # code was 91). Full clone above ⇒ rev-list --count is the true count. + - name: Auto versionCode from commit count + working-directory: straw + run: | + set -euo pipefail + COUNT=$(git rev-list --count HEAD) + VC=$(( COUNT - 12208 )) + sed -i "s/^const val STRAW_VERSION_CODE = .*/const val STRAW_VERSION_CODE = $VC/" \ + buildSrc/src/main/kotlin/ProjectConfig.kt + grep -q "STRAW_VERSION_CODE = $VC\$" buildSrc/src/main/kotlin/ProjectConfig.kt \ + || { echo "::error::versionCode auto-patch failed (sed matched nothing)"; exit 1; } + echo "auto versionCode = $VC (commit count $COUNT)" + + - name: Assemble debug APK + working-directory: straw + env: + STRAW_KEYSTORE_FILE: ${{ github.workspace }}/straw.keystore + STRAW_KEYSTORE_PASS: android + STRAW_KEY_ALIAS: androiddebugkey + STRAW_KEY_PASS: android + # Dogfood log-shipping: the app is compile-DISABLED unless this is set + # (empty ⇒ no crash handler, no network, Settings row hidden). Repo + # secret; absent on contributor forks so their builds don't phone home. + STRAW_LOGS_INGEST_TOKEN: ${{ secrets.STRAW_LOGS_INGEST_TOKEN }} + # Keep the 4-ABI cross-compile off the container rootfs. + CARGO_TARGET_DIR: ${{ github.workspace }}/cargo-target + # ionice idle class + low nice so the build yields disk/CPU to the + # live DBs on Lucy — a 5GB build I/O-starved the MAS Postgres + # checkpoint and dropped Matrix for ~2min on 2026-06-19. + run: ionice -c3 nice -n 19 ./gradlew :strawApp:assembleDebug --no-daemon --stacktrace + + - name: Verify signer + stage APK + working-directory: straw + run: | + set -euo pipefail + VC=$(grep STRAW_VERSION_CODE buildSrc/src/main/kotlin/ProjectConfig.kt | grep -o '[0-9]\+') + APK=$(ls strawApp/build/outputs/apk/debug/*.apk | head -1) + NAME="com.sulkta.straw.debug_${VC}.apk" + cp "$APK" "$GITHUB_WORKSPACE/$NAME" + echo "Built vc=$VC -> $NAME" + # The whole series is signed with SHA-1 bb9ca96b...; fail loudly if a + # build ever produces a different signer (would break in-place updates). + # apksigner lives under build-tools//. AGP (compileSdk 36) needs + # build-tools 36; older straw-build images pre-installed only 34.0.0 and + # AGP auto-fetched 36 at build time, so this sort -V | tail -1 resolves to + # 36.0.0's apksigner. (ci/Dockerfile now pre-installs 36 too — see there.) + # apksigner is a shell wrapper that needs `java` on PATH; the image + # sets JAVA_HOME but doesn't put its bin on PATH for run steps (gradle + # uses JAVA_HOME directly, so the build itself is fine). + export PATH="$JAVA_HOME/bin:$PATH" + APKSIGNER=$(ls "$ANDROID_HOME"/build-tools/*/apksigner | sort -V | tail -1) + FP=$("$APKSIGNER" verify --print-certs "$APK" | grep -i 'SHA-1' | grep -o '[0-9a-f]\{40\}') + echo "signer SHA-1: $FP" + if [ "$FP" != "bb9ca96b10ebbc1ac48e037a21f350415d18915f" ]; then + echo "::error::APK signer $FP != canonical key — refusing to publish"; exit 1 + fi + echo "STRAW_VC=$VC" >> "$GITHUB_ENV" + echo "STRAW_APK=$NAME" >> "$GITHUB_ENV" + + # ---- Publish (rootless, no docker socket): hand the signed APK to the + # Lucy host forced-command. The host re-verifies the signer, re-signs + # the fdroid index (keystore stays on Lucy), and rsyncs to Rackham. ---- + - name: Publish to fdroid via Lucy host forced-command + # Publish on a push to main OR a manual dispatch. A workflow_dispatch is + # how we ship a strawcore-only change (strawcore is cloned fresh at build + # time) into an APK without any straw code change — without the dispatch + # arm a dispatch builds + verifies the APK but silently never publishes. + if: >- + (github.event_name == 'push' && github.ref == 'refs/heads/main') || + github.event_name == 'workflow_dispatch' + env: + LUCY_KEY: ${{ secrets.STRAW_FDROID_LUCY_KEY }} + # Publish target + its host-key are Forgejo secrets, NOT literals, so + # this public workflow carries no infra topology — only references. + PUBLISH_TARGET: ${{ secrets.STRAW_PUBLISH_TARGET }} + LUCY_HOSTKEY: ${{ secrets.STRAW_LUCY_HOSTKEY }} + run: | + set -euo pipefail + install -d -m700 "$HOME/.ssh" + printf '%s\n' "$LUCY_KEY" > "$HOME/.ssh/id_lucy" + chmod 600 "$HOME/.ssh/id_lucy" + # Pin the publish host key (no TOFU) — from a secret, not a literal. + printf '%s\n' "$LUCY_HOSTKEY" > "$HOME/.ssh/known_hosts_lucy" + # Hand off the signed APK on stdin. Host exit codes: 0 = published, + # 3 = already published (benign no-op on a same-commit re-run), + # anything else = real failure. + set +e + ssh -i "$HOME/.ssh/id_lucy" -o IdentitiesOnly=yes -o BatchMode=yes \ + -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$HOME/.ssh/known_hosts_lucy" \ + "$PUBLISH_TARGET" "publish ${STRAW_APK}" < "$GITHUB_WORKSPACE/${STRAW_APK}" + rc=$? + set -e + case "$rc" in + 0) echo "Published vc=${STRAW_VC} to fdroid.sulkta.com (rootless — no docker socket)";; + 3) echo "vc=${STRAW_VC} already published — nothing to do";; + *) echo "::error::fdroid publish failed (rc=$rc)"; exit "$rc";; + esac diff --git a/.github/changed-lines-count-labeler.yml b/.github/changed-lines-count-labeler.yml deleted file mode 100644 index 902f376c0..000000000 --- a/.github/changed-lines-count-labeler.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Add 'size/small' label to any changes with less than 50 lines -size/small: - max: 49 - -# Add 'size/medium' label to any changes between 50 and 249 lines -size/medium: - min: 50 - max: 249 - -# Add 'size/large' label to any changes between 250 and 749 lines -size/large: - min: 250 - max: 749 - -# Add 'size/giant' label to any changes for more than 749 lines -size/giant: - min: 750 diff --git a/.github/workflows/backport-pr.yml b/.github/workflows/backport-pr.yml deleted file mode 100644 index 1e3074064..000000000 --- a/.github/workflows/backport-pr.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Backport merged pull request -on: - issue_comment: - types: [created] -permissions: - contents: write # for comment creation on original PR - pull-requests: write -jobs: - backport: - name: Backport pull request - runs-on: ubuntu-latest - - # Only run when the comment starts with the `/backport` command on a PR and - # the commenter has write access to the repository. We do not want to allow - # everybody to trigger backports and create branches in our repository. - if: > - github.event.issue.pull_request && - startsWith(github.event.comment.body, '/backport ') && - ( - github.event.comment.author_association == 'OWNER' || - github.event.comment.author_association == 'COLLABORATOR' || - github.event.comment.author_association == 'MEMBER' - ) - steps: - - uses: actions/checkout@v6 - - name: Get backport metadata - # the target branch is the first argument after `/backport` - env: - COMMENT_BODY: ${{ github.event.comment.body }} - run: | - set -euo pipefail - body="$COMMENT_BODY" - - line=${body%%$'\n'*} # Get the first line - if [[ $line =~ ^/backport[[:space:]]+([^[:space:]]+) ]]; then - echo "BACKPORT_TARGET=${BASH_REMATCH[1]}" >> "$GITHUB_ENV" - else - echo "Usage: /backport " >&2 - exit 1 - fi - - - name: Create backport pull request - uses: korthout/backport-action@v4 - with: - add_labels: 'backport' - copy_labels_pattern: '.*' - label_pattern: '' - target_branches: ${{ env.BACKPORT_TARGET }} \ No newline at end of file diff --git a/.github/workflows/build-release-apk.yml b/.github/workflows/build-release-apk.yml deleted file mode 100644 index 52c3aeb29..000000000 --- a/.github/workflows/build-release-apk.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: "Build unsigned release APK on master" - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - ref: 'master' - - - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '21' - cache: 'gradle' - - - name: "Build release APK" - run: ./gradlew assembleRelease --stacktrace - - - name: "Rename APK" - run: | - VERSION_NAME="$(jq -r ".elements[0].versionName" "app/build/outputs/apk/release/output-metadata.json")" - echo "Version name: $VERSION_NAME" >> "$GITHUB_STEP_SUMMARY" - echo '```json' >> "$GITHUB_STEP_SUMMARY" - cat "app/build/outputs/apk/release/output-metadata.json" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" - # assume there is only one APK in that folder - mv app/build/outputs/apk/release/*.apk "app/build/outputs/apk/release/NewPipe_v$VERSION_NAME.apk" - - - name: "Upload APK" - uses: actions/upload-artifact@v7 - with: - name: app - path: app/build/outputs/apk/release/*.apk diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 87a3ca33c..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: CI - -on: - workflow_dispatch: - pull_request: - branches: - - dev - - master - - refactor - - release** - paths-ignore: - - 'README.md' - - 'doc/**' - - 'fastlane/**' - - 'assets/**' - - '.github/**/*.md' - - '.github/FUNDING.yml' - - '.github/ISSUE_TEMPLATE/**' - push: - branches: - - dev - - master - paths-ignore: - - 'README.md' - - 'doc/**' - - 'fastlane/**' - - 'assets/**' - - '.github/**/*.md' - - '.github/FUNDING.yml' - - '.github/ISSUE_TEMPLATE/**' - -jobs: - build-and-test-jvm: - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - uses: actions/checkout@v6 - - uses: gradle/actions/wrapper-validation@v6 - - - name: create and checkout branch - # push events already checked out the branch - if: github.event_name == 'pull_request' - env: - BRANCH: ${{ github.head_ref }} - run: git checkout -B "$BRANCH" - - - name: set up JDK - uses: actions/setup-java@v5 - with: - java-version: 21 - distribution: "temurin" - cache: 'gradle' - - - name: Build debug APK and run jvm tests - run: ./gradlew assembleDebug lintDebug testDebugUnitTest --stacktrace -DskipFormatKtlint - - - name: Upload APK - uses: actions/upload-artifact@v7 - with: - name: app - path: app/build/outputs/apk/debug/*.apk - - test-android: - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - matrix: - include: - - api-level: 23 - target: default - arch: x86 - - api-level: 35 - target: default - arch: x86_64 - - permissions: - contents: read - - steps: - - uses: actions/checkout@v6 - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: set up JDK - uses: actions/setup-java@v5 - with: - java-version: 21 - distribution: "temurin" - cache: 'gradle' - - - name: Run android tests - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: ${{ matrix.api-level }} - target: ${{ matrix.target }} - arch: ${{ matrix.arch }} - script: ./gradlew connectedCheck --stacktrace - - - name: Upload test report when tests fail # because the printed out stacktrace (console) is too short, see also #7553 - uses: actions/upload-artifact@v7 - if: failure() - with: - name: android-test-report-api${{ matrix.api-level }} - path: app/build/reports/androidTests/connected/** - - sonar: - if: ${{ false }} # the key has expired and needs to be regenerated by the sonar admins - runs-on: ubuntu-latest - - permissions: - contents: read - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: Set up JDK - uses: actions/setup-java@v5 - with: - java-version: 21 - distribution: "temurin" - cache: 'gradle' - - - name: Cache SonarCloud packages - uses: actions/cache@v5 - with: - path: ~/.sonar/cache - key: ${{ runner.os }}-sonar - restore-keys: ${{ runner.os }}-sonar - - - name: Build and analyze - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: ./gradlew build sonar --info diff --git a/.github/workflows/image-minimizer.js b/.github/workflows/image-minimizer.js deleted file mode 100644 index 0a1e56c56..000000000 --- a/.github/workflows/image-minimizer.js +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Script for minimizing big images (jpg,gif,png) when they are uploaded to GitHub and not edited otherwise - */ -module.exports = async ({github, context}) => { - const IGNORE_KEY = ''; - const IGNORE_ALT_NAME_END = 'ignoreImageMinify'; - // Targeted maximum height - const IMG_MAX_HEIGHT_PX = 600; - // maximum width of GitHub issues/comments - const IMG_MAX_WIDTH_PX = 800; - // all images that have a lower aspect ratio (-> have a smaller width) than this will be minimized - const MIN_ASPECT_RATIO = IMG_MAX_WIDTH_PX / IMG_MAX_HEIGHT_PX - - // Get the body of the image - let initialBody = null; - if (context.eventName == 'issue_comment') { - initialBody = context.payload.comment.body; - } else if (context.eventName == 'issues') { - initialBody = context.payload.issue.body; - } else if (context.eventName == 'pull_request') { - initialBody = context.payload.pull_request.body; - } else { - console.log('Aborting: No body found'); - return; - } - console.log(`Found body: \n${initialBody}\n`); - - // Check if we should ignore the currently processing element - if (initialBody.includes(IGNORE_KEY)) { - console.log('Ignoring: Body contains IGNORE_KEY'); - return; - } - - // Regex for finding images (simple variant) ![ALT_TEXT](https://*.githubusercontent.com//.) - const REGEX_USER_CONTENT_IMAGE_LOOKUP = /\!\[([^\]]*)\]\((https:\/\/[-a-z0-9]+\.githubusercontent\.com\/\d+\/[-0-9a-f]{32,512}\.(jpg|gif|png))\)/gm; - const REGEX_ASSETS_IMAGE_LOOKUP = /\!\[([^\]]*)\]\((https:\/\/github\.com\/(?:user-attachments\/assets|[-\w\d]+\/[-\w\d]+\/assets\/\d+)\/[\-0-9a-f]{32,512})\)/gm; - - // Check if we found something - let foundSimpleImages = REGEX_USER_CONTENT_IMAGE_LOOKUP.test(initialBody) - || REGEX_ASSETS_IMAGE_LOOKUP.test(initialBody); - if (!foundSimpleImages) { - console.log('Found no simple images to process'); - return; - } - - console.log('Found at least one simple image to process'); - - // Require the probe lib for getting the image dimensions - const probe = require('probe-image-size'); - - var wasMatchModified = false; - - // Try to find and replace the images with minimized ones - let newBody = await replaceAsync(initialBody, REGEX_USER_CONTENT_IMAGE_LOOKUP, minimizeAsync); - newBody = await replaceAsync(newBody, REGEX_ASSETS_IMAGE_LOOKUP, minimizeAsync); - - if (!wasMatchModified) { - console.log('Nothing was modified. Skipping update'); - return; - } - - // Update the corresponding element - if (context.eventName == 'issue_comment') { - console.log('Updating comment with id', context.payload.comment.id); - await github.rest.issues.updateComment({ - comment_id: context.payload.comment.id, - owner: context.repo.owner, - repo: context.repo.repo, - body: newBody - }) - } else if (context.eventName == 'issues') { - console.log('Updating issue', context.payload.issue.number); - await github.rest.issues.update({ - issue_number: context.payload.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: newBody - }); - } else if (context.eventName == 'pull_request') { - console.log('Updating pull request', context.payload.pull_request.number); - await github.rest.pulls.update({ - pull_number: context.payload.pull_request.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: newBody - }); - } - - // Async replace function from https://stackoverflow.com/a/48032528 - async function replaceAsync(str, regex, asyncFn) { - const promises = []; - str.replace(regex, (match, ...args) => { - const promise = asyncFn(match, ...args); - promises.push(promise); - }); - const data = await Promise.all(promises); - return str.replace(regex, () => data.shift()); - } - - async function minimizeAsync(match, g1, g2) { - console.log(`Found match '${match}'`); - - if (g1.endsWith(IGNORE_ALT_NAME_END)) { - console.log(`Ignoring match '${match}': IGNORE_ALT_NAME_END`); - return match; - } - - let probeAspectRatio = 0; - let shouldModify = false; - try { - console.log(`Probing ${g2}`); - let probeResult = await probe(g2); - if (probeResult == null) { - throw 'No probeResult'; - } - if (probeResult.hUnits != 'px') { - throw `Unexpected probeResult.hUnits (expected px but got ${probeResult.hUnits})`; - } - if (probeResult.height <= 0) { - throw `Unexpected probeResult.height (height is invalid: ${probeResult.height})`; - } - if (probeResult.wUnits != 'px') { - throw `Unexpected probeResult.wUnits (expected px but got ${probeResult.wUnits})`; - } - if (probeResult.width <= 0) { - throw `Unexpected probeResult.width (width is invalid: ${probeResult.width})`; - } - console.log(`Probing resulted in ${probeResult.width}x${probeResult.height}px`); - - probeAspectRatio = probeResult.width / probeResult.height; - shouldModify = probeResult.height > IMG_MAX_HEIGHT_PX && probeAspectRatio < MIN_ASPECT_RATIO; - } catch(e) { - console.log('Probing failed:', e); - // Immediately abort - return match; - } - - if (shouldModify) { - wasMatchModified = true; - console.log(`Modifying match '${match}'`); - return `${g1}`; - } - - console.log(`Match '${match}' is ok/will not be modified`); - return match; - } -} diff --git a/.github/workflows/image-minimizer.yml b/.github/workflows/image-minimizer.yml deleted file mode 100644 index 15c2aacaf..000000000 --- a/.github/workflows/image-minimizer.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Image Minimizer - -on: - issue_comment: - types: [created, edited] - issues: - types: [opened, edited] - pull_request: - types: [opened, edited] - -permissions: - issues: write - pull-requests: write - -jobs: - try-minimize: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: 16 - - - name: Install probe-image-size - run: npm i probe-image-size@7.2.3 --ignore-scripts - - - name: Minimize simple images - uses: actions/github-script@v9 - timeout-minutes: 3 - with: - script: | - const script = require('.github/workflows/image-minimizer.js'); - await script({github, context}); diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml deleted file mode 100644 index b3495135f..000000000 --- a/.github/workflows/no-response.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: No Response - -# Both `issue_comment` and `scheduled` event types are required for this Action -# to work properly. -on: - issue_comment: - types: [created] - schedule: - # Run daily at midnight. - - cron: '0 0 * * *' - -permissions: - issues: write - pull-requests: write - -jobs: - noResponse: - runs-on: ubuntu-latest - steps: - - uses: lee-dohm/no-response@v0.5.0 - with: - token: ${{ github.token }} - daysUntilClose: 14 - responseRequiredLabel: waiting for author diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml deleted file mode 100644 index a18daca3a..000000000 --- a/.github/workflows/pr-labeler.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: "PR size labeler" -on: [pull_request_target] -permissions: - contents: read - pull-requests: write - -jobs: - changed-lines-count-labeler: - runs-on: ubuntu-latest - name: Automatically labelling pull requests based on the changed lines count - permissions: - pull-requests: write - steps: - - name: Set a label - uses: TeamNewPipe/changed-lines-count-labeler@main - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - configuration-path: .github/changed-lines-count-labeler.yml diff --git a/buildSrc/src/main/kotlin/ProjectConfig.kt b/buildSrc/src/main/kotlin/ProjectConfig.kt index 42b67dc1e..6a6042811 100644 --- a/buildSrc/src/main/kotlin/ProjectConfig.kt +++ b/buildSrc/src/main/kotlin/ProjectConfig.kt @@ -9,6 +9,53 @@ const val STRAW_SDK_TARGET = 35 // Sulkta fork — Straw // +// vc=91 / 0.1.0-CY — reliability + security fix sprint (straw audit 2026-07-04): +// * The in-app updater no longer self-bricks on a routine cert renewal. It +// used to pin the fdroid.sulkta.com leaf SPKI + the LE "E7" intermediate, +// but the leaf rotates every ~90-day renewal and LE rotates intermediates +// from a pool (they explicitly say don't pin them), so a normal renewal was +// GUARANTEED to miss both pins — silently killing the ONLY update path, +// with the fix reachable only through the channel the break just killed. +// Dropped pinning: default system-CA validation still blocks a MITM, and a +// swapped APK can't install over Straw anyway (the package installer +// verifies it against our signing key). Also: the check now distinguishes +// "couldn't reach the index" from "up to date" (no more "checked just now / +// up to date" while actually blind), added a 30s call timeout + a bounded +// index read, guarded the API-26 NotificationChannel (was a crash on 7.x), +// and setOnlyAlertOnce so a pending update doesn't re-nag every tick. +// * Android 13+ update/media notifications actually appear now: +// POST_NOTIFICATIONS was manifest-declared but never requested at runtime, +// so nm.notify() was a silent no-op on 13+. Requested once on cold start. +// * The published APK is no longer debuggable (isDebuggable=false on the +// shipped debug variant) — closes an ADB/USB `run-as` dump of watch/search +// history + the subscription list. No package-id cutover; updates in place. +// * Crash fixes: (a) opening a video whose channel repeats a videoId in its +// first ~20 entries hard-crashed Compose on a duplicate LazyColumn key — +// moreFromChannel + related now distinctBy url; (b) in-app back died +// permanently after one back-to-Home on Android 12+ (moveTaskToBack doesn't +// destroy the activity, so the disabled callback never re-enabled) — the +// callback is now driven by live nav depth instead of disable-and-redispatch; +// (c) the local-playlists store was uncapped, so a hostile NewPipe import +// could persist a huge blob that ANR'd every cold start — added per-store +// caps (playlists / items / string length) + off-main hydration like +// ResumePositionsStore; (d) a SponsorBlock fetch that resolves after you +// skip to another video no longer hijacks the now-playing item (staleness +// fence). +// * Cache-size caps that first-launch defaulted to "Unlimited" (100k rows, +// because nearest() was exact-match-or-Unlimited over a set the defaults +// weren't in) now snap up to the nearest finite cap. +// * The RYD + SponsorBlock FFI shims wrap the uniffi call, so a core panic +// (now catchable — see below) is swallowed to null/empty per their contract +// instead of crashing video-detail load. +// * Rust FFI wrapper: release profile is panic="unwind" (was "abort" — abort +// defeated UniFFI's catch_unwind + tokio's spawn_blocking JoinError capture, +// turning ANY panic in the extractor core into a whole-app SIGABRT), and +// every blocking extractor call is wrapped in a 60s wall-clock timeout so a +// wedged JS/HTTP path unblocks the caller instead of spinning forever and +// piling detached threads toward tokio's 512 blocking-pool cap. Pairs with +// the strawcore-core reliability fix (bounded rquickjs 64MiB/2MiB/5s + +// invalidate-on-failure player.js self-heal). +// // vc=90 / 0.1.0-CX — cold-start hydration + status-bar bleed fix + RYD FFI slim: // * ResumePositionsStore + EnrichmentStore no longer JSON-decode on the main // thread in Application.onCreate. Resume was the heaviest cold-start cost in @@ -312,6 +359,10 @@ const val STRAW_SDK_TARGET = 35 // vc=19 / 0.1.0-AE — rust pipeline cutover. Extraction via // strawcore-core (Sulkta-OSS/strawcore) via the UniFFI wrapper; no // NewPipeExtractor in the runtime path. -const val STRAW_VERSION_CODE = 90 -const val STRAW_VERSION_NAME = "0.1.0-CX" +// versionCode is AUTO-DERIVED in CI from the git commit count (see +// .forgejo/workflows/build.yml "Auto versionCode from commit count") so a +// release can never silently reuse a code. This literal is the local-dev +// default only; CI overwrites it at build time. +const val STRAW_VERSION_CODE = 92 +const val STRAW_VERSION_NAME = "0.1.0-CZ" const val STRAW_APPLICATION_ID = "com.sulkta.straw" diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fcf2ab55a..f9c230243 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ bridge = "v2.0.2" cardview = "1.0.0" checkstyle = "13.4.2" coil = "3.4.0" -compose = "1.11.3" +compose = "1.11.4" constraintlayout = "2.2.1" core = "1.18.0" coroutines = "1.11.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2f1ce3924..b7f4aaa95 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,9 +21,31 @@ repository = "https://git.sulkta.com/Sulkta-OSS/straw" strip = true lto = "thin" codegen-units = 1 -panic = "abort" +# unwind (NOT abort): panic=abort defeats UniFFI's catch_unwind AND tokio's +# spawn_blocking JoinError capture, turning ANY panic in the extractor core +# (which chews weekly-rotating, attacker-influenced YouTube payloads) into an +# instant whole-app SIGABRT. unwind restores both safety nets so a core panic +# surfaces as a caught StrawcoreException instead of crashing the app. The +# unwind-table size cost is a rounding error against the 4-ABI jniLibs payload. +# (straw + FFI-wrapper audits, 2026-07-04.) +panic = "unwind" opt-level = "z" +# Per-package opt-level overrides (Straw speed audit, S5). The whole tree stays +# size-optimized ("z"), but the JS-interpreter + parsing hot paths run measurably +# slow at "z" — QuickJS's C bytecode interpreter, the regex DFA engine, and JSON +# deserialization are all tight inner loops where "z" trades real runtime for a +# few KB. Bumping ONLY these three to opt-level = 2 bounds the APK-size cost to +# these crates while restoring interpreter/parse throughput. Crate names verified +# present in Cargo.lock: rquickjs-sys (QuickJS C interpreter, compiled via cc), +# regex-automata (regex execution engine, pulled by `regex`), serde_json. +[profile.release.package.rquickjs-sys] +opt-level = 2 +[profile.release.package.regex-automata] +opt-level = 2 +[profile.release.package.serde_json] +opt-level = 2 + # `url` crate for video-id extraction in stream.rs. [workspace.dependencies] url = "2" diff --git a/rust/strawcore/src/channel.rs b/rust/strawcore/src/channel.rs index 6f9554f66..71cf43610 100644 --- a/rust/strawcore/src/channel.rs +++ b/rust/strawcore/src/channel.rs @@ -33,11 +33,8 @@ pub async fn channel_info(input: String) -> Result log::info!("strawcore::channel_info input_len={}", input.len()); crate::runtime::ensure_initialized(); let identifier = resolve_channel_identifier(&input)?; - let core = tokio::task::spawn_blocking(move || core_channel_info(identifier)) - .await - .map_err(|e| StrawcoreError::Extractor { - msg: format!("join: {e}"), - })??; + let core = crate::runtime::run_extract("channel_info", move || core_channel_info(identifier)) + .await?; Ok(map_channel(core)) } @@ -63,11 +60,10 @@ pub async fn channel_videos_continuation(token: String) -> Result Opt } buf.extend_from_slice(&chunk); } - // Lossy decode: a strict from_utf8 would drop the whole response on a - // single mojibake byte; serde_json tolerates U+FFFD in string values. - Some(String::from_utf8_lossy(&buf).into_owned()) + // Prefer a zero-copy move on valid UTF-8 (the common case); fall back to a + // lossy copy only on a mojibake byte (a strict from_utf8 would otherwise + // drop the whole response — serde_json tolerates U+FFFD in string values). + // Mirrors the core downloader's default_impl fast path (S4). + Some(match String::from_utf8(buf) { + Ok(s) => s, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + }) } // --------------------------------------------------------------------------- diff --git a/rust/strawcore/src/runtime.rs b/rust/strawcore/src/runtime.rs index 336d6b4bb..eb5fc40aa 100644 --- a/rust/strawcore/src/runtime.rs +++ b/rust/strawcore/src/runtime.rs @@ -17,12 +17,14 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use strawcore_core::downloader::ReqwestDownloader; use strawcore_core::localization::{ContentCountry, Localization}; use strawcore_core::newpipe::NewPipe; +use crate::error::StrawcoreError; + static INITIALIZED: AtomicBool = AtomicBool::new(false); static LAST_ATTEMPT_MS: AtomicU64 = AtomicU64::new(0); // Guards the actual init attempt so concurrent calls don't all try @@ -94,3 +96,47 @@ pub fn ensure_initialized() { } } } + +/// Wall-clock ceiling on a single blocking extractor call. Core bounds each +/// HTTP request to 30s and (since the 2026-07-04 reliability fix) the JS +/// deobfuscation runtime to a 5s interrupt, so a legitimate multi-round-trip +/// extraction on a slow connection stays comfortably under this; anything +/// past it is a wedge (a pathological player.js, a black-holed CDN) and we'd +/// rather surface a timeout to the Kotlin caller than spin forever. +const EXTRACT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Run a blocking strawcore-core call on the tokio blocking pool, bounded by +/// [`EXTRACT_TIMEOUT`]. Replaces the raw `spawn_blocking(..).await??` pattern +/// at every FFI entry point so a hung extraction unblocks the Kotlin suspend +/// instead of (a) spinning the spinner forever and (b) piling detached +/// blocking threads toward tokio's 512-thread cap, after which *all* +/// spawn_blocking-based FFI calls would queue behind the wedged ones for the +/// process lifetime. +/// +/// Caveat (documented, not a bug): `spawn_blocking` tasks are not abortable, +/// so on timeout the detached thread keeps running until the core call +/// returns on its own — core's per-request (30s) + JS-interrupt (5s) bounds +/// cap how long that is. The win here is unblocking the *caller* and bounding +/// retry pileup, not reclaiming the thread instantly. +pub(crate) async fn run_extract(what: &'static str, f: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, + E: Send + 'static, + StrawcoreError: From, +{ + match tokio::time::timeout(EXTRACT_TIMEOUT, tokio::task::spawn_blocking(f)).await { + // Timed out waiting on the blocking task. + Err(_elapsed) => Err(StrawcoreError::Extractor { + msg: format!("timeout: {what} exceeded {}s", EXTRACT_TIMEOUT.as_secs()), + }), + // Blocking task finished within the deadline but the join failed — + // panic (only under a non-abort profile) or cancellation. + Ok(Err(join)) => Err(StrawcoreError::Extractor { + msg: format!("join: {join}"), + }), + // Blocking task returned; propagate its own error via the existing + // `From` mapping. + Ok(Ok(inner)) => inner.map_err(StrawcoreError::from), + } +} diff --git a/rust/strawcore/src/search.rs b/rust/strawcore/src/search.rs index 308bacdda..726019a96 100644 --- a/rust/strawcore/src/search.rs +++ b/rust/strawcore/src/search.rs @@ -84,13 +84,10 @@ pub async fn search(query: String) -> Result { // the hot entry points. Now every extractor entry re-asserts // — cheap when INITIALIZED is true (single Acquire load). crate::runtime::ensure_initialized(); - let result = tokio::task::spawn_blocking(move || { + let result = crate::runtime::run_extract("search", move || { search_extractor::search(&query, SearchFilter::Videos) }) - .await - .map_err(|e| StrawcoreError::Extractor { - msg: format!("join: {e}"), - })??; + .await?; // Page 1 carries the first continuation token (None once YT stops // handing them out). The Kotlin SearchViewModel passes it back to // `search_continuation` on scroll. @@ -106,10 +103,9 @@ pub async fn search(query: String) -> Result { pub async fn search_continuation(token: String) -> Result { log::info!("strawcore::search_continuation token_len={}", token.len()); crate::runtime::ensure_initialized(); - let page = tokio::task::spawn_blocking(move || search_extractor::search_continuation(&token)) - .await - .map_err(|e| StrawcoreError::Extractor { - msg: format!("join: {e}"), - })??; + let page = crate::runtime::run_extract("search_continuation", move || { + search_extractor::search_continuation(&token) + }) + .await?; Ok(page_from_core(page)) } diff --git a/rust/strawcore/src/stream.rs b/rust/strawcore/src/stream.rs index e95c313ea..09eb19cfd 100644 --- a/rust/strawcore/src/stream.rs +++ b/rust/strawcore/src/stream.rs @@ -78,11 +78,10 @@ pub async fn stream_info(input: String) -> Result { crate::runtime::ensure_initialized(); let video_id = resolve_video_id(&input)?; let video_id_for_call = video_id.clone(); - let core = tokio::task::spawn_blocking(move || core_stream_info(&video_id_for_call)) - .await - .map_err(|e| StrawcoreError::Extractor { - msg: format!("join: {e}"), - })??; + let core = crate::runtime::run_extract("stream_info", move || { + core_stream_info(&video_id_for_call) + }) + .await?; Ok(map_stream_info(video_id, core)) } @@ -95,11 +94,10 @@ pub async fn stream_metadata(input: String) -> Result<(i64, i64), StrawcoreError log::info!("strawcore::stream_metadata input_len={}", input.len()); crate::runtime::ensure_initialized(); let video_id = resolve_video_id(&input)?; - let core = tokio::task::spawn_blocking(move || core_stream_metadata(&video_id)) - .await - .map_err(|e| StrawcoreError::Extractor { - msg: format!("join: {e}"), - })??; + let core = crate::runtime::run_extract("stream_metadata", move || { + core_stream_metadata(&video_id) + }) + .await?; Ok((clamp_nonneg(core.view_count), core.duration_seconds.max(0))) } diff --git a/strawApp/build.gradle.kts b/strawApp/build.gradle.kts index bf1bdbed2..8d13d14cf 100644 --- a/strawApp/build.gradle.kts +++ b/strawApp/build.gradle.kts @@ -10,6 +10,7 @@ */ import com.android.build.api.dsl.ApplicationExtension +import java.util.Properties plugins { alias(libs.plugins.android.application) @@ -36,6 +37,30 @@ configure { versionCode = STRAW_VERSION_CODE versionName = STRAW_VERSION_NAME resValue("string", "app_name", "Straw") + + // Dogfood log-shipping token (feature/diag/LogShipper.kt). This repo + // is PUBLIC — the token must NEVER be committed, so it's injected at + // build time and defaults to "": + // 1. env var STRAW_LOGS_INGEST_TOKEN — CI: add a Forgejo Actions + // repo secret STRAW_LOGS_INGEST_TOKEN and pass it in the + // "Assemble debug APK" step's env block in + // .forgejo/workflows/build.yml. + // 2. `straw.logs.ingest.token=…` in local.properties (gitignored) + // for local dogfood builds. + // Empty token ⇒ LogShipper is fully disabled (no-op, zero network, + // no Settings row) — contributor builds neither break nor phone home. + val strawLogsToken: String = System.getenv("STRAW_LOGS_INGEST_TOKEN") + ?: run { + val props = Properties() + val f = rootProject.file("local.properties") + if (f.exists()) f.inputStream().use { props.load(it) } + props.getProperty("straw.logs.ingest.token") + } + ?: "" + // Escape so a pathological token can't break out of the generated + // String literal in BuildConfig.java. + val escapedToken = strawLogsToken.replace("\\", "\\\\").replace("\"", "\\\"") + buildConfigField("String", "STRAW_LOGS_TOKEN", "\"$escapedToken\"") } // Explicit signing so CI / release builds reuse ONE keystore instead of @@ -66,7 +91,15 @@ configure { // strawApp/proguard-rules.pro cover UniFFI + JNA + // kotlinx-serialization companions. debug { - isDebuggable = true + // NOT debuggable, even though this is the `debug` buildType: the + // debug variant is what we PUBLISH to fdroid.sulkta.com, and a + // shipped debuggable APK lets any ADB/USB attacker `run-as` the + // package and dump watch/search history + the full subscription + // list — defeating allowBackup=false / dataExtractionRules on a + // privacy-focused client. Flipping this needs no package-id + // cutover: signing + the `.debug` suffix below are unchanged, so + // in-place fdroid updates keep working. + isDebuggable = false // Keep the `.debug` applicationId suffix (package stays // com.sulkta.straw.debug) so in-place fdroid updates + the in-app // auto-updater keep working and nobody loses their subs/history. diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawActivity.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawActivity.kt index fbc39d27c..32a326fb5 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawActivity.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawActivity.kt @@ -5,12 +5,16 @@ package com.sulkta.straw +import android.Manifest import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build import android.os.Bundle import androidx.activity.ComponentActivity import androidx.activity.OnBackPressedCallback import androidx.activity.compose.setContent import androidx.activity.enableEdgeToEdge +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.isSystemInDarkTheme import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize @@ -20,6 +24,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.SideEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -27,6 +32,7 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.core.content.ContextCompat import androidx.media3.common.util.UnstableApi import com.sulkta.straw.data.Settings import com.sulkta.straw.data.ThemeMode @@ -62,10 +68,23 @@ class StrawActivity : ComponentActivity() { */ private val pendingDeepLink = MutableStateFlow(null) + /** + * Android 13+ gates every `notify()` (media controls + the + * update-available notice) behind a runtime POST_NOTIFICATIONS grant. + * Registered here (before the activity is STARTED, as the API requires) + * and launched once on cold start. The result is intentionally ignored: + * Android won't re-prompt after a permanent choice, and both playback + * and the update surface degrade gracefully without it (the passive + * Settings "Update available" text still shows). + */ + private val notificationPermissionLauncher = + registerForActivityResult(ActivityResultContracts.RequestPermission()) { } + @OptIn(UnstableApi::class) override fun onCreate(savedInstanceState: Bundle?) { enableEdgeToEdge() super.onCreate(savedInstanceState) + maybeRequestNotificationPermission() val startUrl = pickYouTubeUrl(intent) @@ -100,25 +119,39 @@ class StrawActivity : ComponentActivity() { expanded = true } - DisposableEffect(nav) { - val cb = object : OnBackPressedCallback(true) { + // Intercept back ONLY when there's an in-app action to take: + // collapse an expanded player, or pop the browse stack. At + // the browse root with nothing open the callback is disabled + // so the system handles back natively (moveTaskToBack on + // A12+). The previous version disabled the callback and + // re-dispatched to the system at root — but on A12+ + // moveTaskToBack does NOT destroy the activity, so the + // composition (and the now-disabled callback) survived and + // DisposableEffect(nav) never re-ran to re-enable it, killing + // all in-app back permanently after one back-to-Home. + // Driving isEnabled from live nav/expanded state fixes that. + val canGoBack = expanded || nav.stack.size > 1 + val backCallback = remember { + object : OnBackPressedCallback(canGoBack) { override fun handleOnBackPressed() { - // Back order: the true-fullscreen Player pops - // first; an expanded player collapses to the - // minibar; otherwise pop the browse stack (or - // exit at root). + // The true-fullscreen Player is a nav screen, so + // it pops via the stack branch; an expanded + // (non-fullscreen) player collapses first. if (nav.current !is Screen.Player && expanded) { expanded = false - return - } - if (!nav.pop()) { - isEnabled = false - this@StrawActivity.onBackPressedDispatcher.onBackPressed() + } else { + // Guaranteed poppable: enabled only when + // expanded || stack.size > 1, and the + // expanded case is handled above. + nav.pop() } } } - onBackPressedDispatcher.addCallback(cb) - onDispose { cb.remove() } + } + SideEffect { backCallback.isEnabled = canGoBack } + DisposableEffect(onBackPressedDispatcher) { + onBackPressedDispatcher.addCallback(backCallback) + onDispose { backCallback.remove() } } // Open the deep-linked video into the expandable player on @@ -151,7 +184,23 @@ class StrawActivity : ComponentActivity() { expandedTarget = expanded, onTargetChange = { expanded = it }, onFullscreen = { url, title -> nav.push(Screen.Player(url, title)) }, - onOpenChannel = { url, name -> nav.push(Screen.Channel(url, name)) }, + // Navigating to a DIFFERENT destination from the + // expanded player must collapse it to the minibar + // first — the expanded body is opaque and sits + // z-above ScreenContent, so without this the + // pushed Channel screen is invisible until the + // user manually minimizes the player. Playback + // continues in the minibar (collapse never + // touches the controller). Related-video taps + // are the OPPOSITE case: they go through + // openVideo (swap-in-place, sets expanded=true) + // and stay expanded; ⛶ fullscreen also stays + // expanded (Screen.Player un-composes this + // overlay and restores it expanded on pop). + onOpenChannel = { url, name -> + expanded = false + nav.push(Screen.Channel(url, name)) + }, onOpenVideo = openVideo, ) } @@ -173,6 +222,18 @@ class StrawActivity : ComponentActivity() { pickYouTubeUrl(intent)?.let { pendingDeepLink.value = it } } + /** Ask for POST_NOTIFICATIONS on API 33+ if it isn't already granted. */ + private fun maybeRequestNotificationPermission() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) return + val granted = ContextCompat.checkSelfPermission( + this, + Manifest.permission.POST_NOTIFICATIONS, + ) == PackageManager.PERMISSION_GRANTED + if (!granted) { + notificationPermissionLauncher.launch(Manifest.permission.POST_NOTIFICATIONS) + } + } + @Composable private fun ScreenContent( nav: Navigator, diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt index e1922b988..e98f4f64a 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/StrawApp.kt @@ -21,6 +21,7 @@ import com.sulkta.straw.data.SearchCache import com.sulkta.straw.data.Settings import com.sulkta.straw.data.Subscriptions import com.sulkta.straw.feature.dataimport.SettingsImport +import com.sulkta.straw.feature.diag.LogShipper import com.sulkta.straw.feature.feed.FeedRefreshScheduler import com.sulkta.straw.feature.update.UpdateScheduler import com.sulkta.straw.feature.update.runUpdateCheck @@ -90,6 +91,12 @@ class StrawApp : Application(), SingletonImageLoader.Factory { override fun onCreate() { super.onCreate() + // Dogfood crash shipping — wrap the default uncaught-exception + // handler FIRST so even an init crash below gets shipped. The + // wrapper always chains to the previous handler (normal crash + // behavior is untouched) and is a no-op in builds without an + // injected STRAW_LOGS_TOKEN. + LogShipper.installCrashHandler(this) // Path C-7: route Rust `log::*` calls into Android logcat under tag // "strawcore". Without this, every log line emitted from rustypipe / // strawcore is silently dropped, making playback regressions invisible diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/data/PlaylistsStore.kt b/strawApp/src/main/kotlin/com/sulkta/straw/data/PlaylistsStore.kt index 22b381868..05fd20840 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/data/PlaylistsStore.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/data/PlaylistsStore.kt @@ -20,6 +20,7 @@ import android.content.SharedPreferences import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.update import kotlinx.coroutines.flow.updateAndGet import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json @@ -45,6 +46,24 @@ data class Playlist( private const val PREFS = "straw_playlists" private const val KEY = "playlists_v1" +// Caps so a hostile/oversized NewPipe import (SettingsImport permitted up to +// 256 playlists × 5000 items with no length bound) can't persist a +// hundreds-of-MB blob that then re-decodes on every cold start → ANR/OOM +// crash-loop recoverable only by clearing app data. Realistic use is a +// handful of playlists with tens–hundreds of short-string items, so these +// bound the pathological case while leaving normal use untouched. +private const val MAX_PLAYLISTS = 100 +private const val MAX_ITEMS_PER_PLAYLIST = 1000 +private const val MAX_NAME_LEN = 100 +private const val MAX_TITLE_LEN = 300 +private const val MAX_URL_LEN = 512 +private const val MAX_UPLOADER_LEN = 100 + +// Refuse to even decode a persisted blob larger than this (chars). A capped +// legitimate export is a few MB at most; anything past this is corrupt or a +// pre-fix poison blob, and we'd rather start empty than OOM decoding it. +private const val MAX_BLOB_CHARS = 8_000_000 + class PlaylistsStore(context: Context) { private val sp: SharedPreferences = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) private val json = Json { ignoreUnknownKeys = true } @@ -53,17 +72,43 @@ class PlaylistsStore(context: Context) { // list so disk converges to the latest in-memory state. private val writer = PrefsWriter(sp) - private val _playlists = MutableStateFlow(load()) + // Seed empty + hydrate OFF the main thread (mirrors ResumePositionsStore). + // The old MutableStateFlow(load()) decoded the whole playlists blob + // synchronously in the constructor, which runs in StrawApp.onCreate on + // Main — a large (even capped) blob blocked cold start. The hydrate shares + // the PrefsWriter FIFO dispatcher (enqueued before this store is published) + // so it always runs before any write's persist, and mutations that arrive + // before it finishes defer themselves (see the `hydrated` guard below). + private val _playlists = MutableStateFlow>(emptyList()) val playlists: StateFlow> = _playlists.asStateFlow() + @Volatile private var hydrated = false + + init { + writer.serial { + try { + val loaded = load() + if (loaded.isNotEmpty()) { + _playlists.update { live -> + // Deferral keeps `live` empty at hydrate time, so this + // is normally just `loaded`; merge-by-id (live wins) is + // the belt-and-suspenders path if a create slipped in. + if (live.isEmpty()) loaded else capPlaylists(mergeById(loaded, live)) + } + } + } finally { + hydrated = true + } + } + } + fun create(name: String): Playlist { val pl = Playlist( id = UUID.randomUUID().toString(), - name = name.trim().ifBlank { "Untitled" }, + name = clampName(name).ifBlank { "Untitled" }, createdAt = System.currentTimeMillis(), ) - _playlists.updateAndGet { it + pl } - persist() + insert(pl) return pl } @@ -76,32 +121,42 @@ class PlaylistsStore(context: Context) { */ fun importPlaylist(name: String, items: List): Playlist { val stampNow = System.currentTimeMillis() - // Dedup within the import + stamp addedAt once. + // Dedup within the import, clamp string lengths, stamp addedAt once, + // and stop at the per-playlist item cap. val seen = HashSet() - val deduped = ArrayList(items.size) - for (it in items) { - if (it.streamUrl.isBlank()) continue - if (!seen.add(it.streamUrl)) continue - deduped.add(it.copy(addedAt = if (it.addedAt == 0L) stampNow else it.addedAt)) + val deduped = ArrayList(minOf(items.size, MAX_ITEMS_PER_PLAYLIST)) + for (raw in items) { + if (raw.streamUrl.isBlank()) continue + val clamped = clampItem(raw).let { if (it.addedAt == 0L) it.copy(addedAt = stampNow) else it } + if (!seen.add(clamped.streamUrl)) continue + deduped.add(clamped) + if (deduped.size >= MAX_ITEMS_PER_PLAYLIST) break } val pl = Playlist( id = UUID.randomUUID().toString(), - name = name.trim().ifBlank { "Untitled" }, + name = clampName(name).ifBlank { "Untitled" }, createdAt = stampNow, items = deduped, ) - _playlists.updateAndGet { it + pl } - persist() + insert(pl) return pl } + private fun insert(pl: Playlist) { + if (!hydrated) { writer.serial { insert(pl) }; return } + _playlists.updateAndGet { capPlaylists(it + pl) } + persist() + } + fun delete(id: String) { + if (!hydrated) { writer.serial { delete(id) }; return } _playlists.updateAndGet { cur -> cur.filterNot { it.id == id } } persist() } fun rename(id: String, newName: String) { - val trimmed = newName.trim().ifBlank { return } + val trimmed = clampName(newName).ifBlank { return } + if (!hydrated) { writer.serial { rename(id, newName) }; return } _playlists.updateAndGet { cur -> cur.map { if (it.id == id) it.copy(name = trimmed) else it } } @@ -109,11 +164,14 @@ class PlaylistsStore(context: Context) { } fun addItem(playlistId: String, item: PlaylistItem) { - val stamped = item.copy(addedAt = System.currentTimeMillis()) + val stamped = clampItem(item).copy(addedAt = System.currentTimeMillis()) + if (stamped.streamUrl.isBlank()) return + if (!hydrated) { writer.serial { addItem(playlistId, item) }; return } _playlists.updateAndGet { cur -> cur.map { pl -> if (pl.id != playlistId) pl else if (pl.items.any { it.streamUrl == stamped.streamUrl }) pl + else if (pl.items.size >= MAX_ITEMS_PER_PLAYLIST) pl else pl.copy(items = pl.items + stamped) } } @@ -121,6 +179,7 @@ class PlaylistsStore(context: Context) { } fun removeItem(playlistId: String, streamUrl: String) { + if (!hydrated) { writer.serial { removeItem(playlistId, streamUrl) }; return } _playlists.updateAndGet { cur -> cur.map { pl -> if (pl.id != playlistId) pl @@ -136,9 +195,37 @@ class PlaylistsStore(context: Context) { writer.write { putString(KEY, json.encodeToString(_playlists.value)) } } + private fun clampName(s: String): String = s.trim().take(MAX_NAME_LEN) + + private fun clampItem(item: PlaylistItem): PlaylistItem = item.copy( + streamUrl = item.streamUrl.take(MAX_URL_LEN), + title = item.title.take(MAX_TITLE_LEN), + thumbnail = item.thumbnail?.take(MAX_URL_LEN), + uploader = item.uploader.take(MAX_UPLOADER_LEN), + ) + + /** Keep the newest MAX_PLAYLISTS (new ones are appended). */ + private fun capPlaylists(list: List): List = + if (list.size <= MAX_PLAYLISTS) list else list.takeLast(MAX_PLAYLISTS) + + /** live entries win on id collision; loaded's remainder is appended. */ + private fun mergeById(loaded: List, live: List): List { + val liveIds = live.mapTo(HashSet()) { it.id } + return live + loaded.filterNot { it.id in liveIds } + } + private fun load(): List = runCatching { val s = sp.getString(KEY, null) ?: return emptyList() + // Don't decode an implausibly huge (corrupt/hostile/legacy-poison) blob. + if (s.length > MAX_BLOB_CHARS) return emptyList() json.decodeFromString>(s) + .takeLast(MAX_PLAYLISTS) + .map { pl -> + pl.copy( + name = clampName(pl.name), + items = pl.items.take(MAX_ITEMS_PER_PLAYLIST).map { clampItem(it) }, + ) + } }.getOrDefault(emptyList()) } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/data/SettingsStore.kt b/strawApp/src/main/kotlin/com/sulkta/straw/data/SettingsStore.kt index 86b3643f6..9fe25bee0 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/data/SettingsStore.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/data/SettingsStore.kt @@ -83,8 +83,19 @@ enum class CacheCap(val label: String, val value: Int) { Unlimited("Unlimited", Int.MAX_VALUE); companion object { + /** + * Resolve a stored/raw cap value to an enum. Snaps UP to the + * smallest offered cap that is >= target, so a value that isn't an + * exact enum member (a legacy default like 20/30/500, or a stored + * value from a build with different granularity) resolves to a real + * FINITE cap that covers it — not silently to Unlimited. Only a + * target larger than every finite cap resolves to Unlimited (which + * is exactly what a stored Int.MAX_VALUE "Unlimited" selection is). + * The prior exact-match-or-Unlimited version turned every non-enum + * default into an effective no-cap (100k rows) on fresh installs. + */ fun nearest(target: Int): CacheCap = - entries.firstOrNull { it.value == target } ?: Unlimited + entries.filter { it.value >= target }.minByOrNull { it.value } ?: Unlimited } } @@ -277,26 +288,29 @@ class SettingsStore(context: Context) { * takes effect immediately (next write trims to the new cap; reads * are unbounded since they're already in memory). * - * Defaults match the earlier hardcoded constants so first-launch - * behavior is unchanged from prior versions. + * Defaults are real enum values (not the old 20/30/500 non-enum + * literals, which — because nearest() used to be exact-match-or-Unlimited + * — silently resolved to Unlimited on every fresh install and made the + * effective cap the hard ceiling). nearest() now snaps up, so even a + * legacy stored non-enum value lands on a finite cap. */ private val _historyWatchesCap = MutableStateFlow( - CacheCap.nearest(sp.getInt(KEY_CACHE_HISTORY_WATCHES, 50)), + CacheCap.nearest(sp.getInt(KEY_CACHE_HISTORY_WATCHES, CacheCap.Tiny.value)), ) val historyWatchesCap: StateFlow = _historyWatchesCap.asStateFlow() private val _historySearchesCap = MutableStateFlow( - loadCap(KEY_CACHE_HISTORY_SEARCHES, default = 20), + loadCap(KEY_CACHE_HISTORY_SEARCHES, default = CacheCap.Tiny.value), ) val historySearchesCap: StateFlow = _historySearchesCap.asStateFlow() private val _resumePositionsCap = MutableStateFlow( - loadCap(KEY_CACHE_RESUME_POSITIONS, default = 500), + loadCap(KEY_CACHE_RESUME_POSITIONS, default = CacheCap.Medium.value), ) val resumePositionsCap: StateFlow = _resumePositionsCap.asStateFlow() private val _searchCacheCap = MutableStateFlow( - loadCap(KEY_CACHE_SEARCH, default = 30), + loadCap(KEY_CACHE_SEARCH, default = CacheCap.Tiny.value), ) val searchCacheCap: StateFlow = _searchCacheCap.asStateFlow() diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt index 979c4dcbc..39657b91e 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt @@ -321,8 +321,11 @@ object SettingsImport { openDb(dbFile).use { db -> val playlistRows = mutableListOf>() // Hard caps so a malicious export with millions of rows - // doesn't walk an unbounded cursor into memory. - db.rawQuery("SELECT uid, name FROM playlists LIMIT 256", null).use { c -> + // doesn't walk an unbounded cursor into memory. Matched to + // PlaylistsStore's MAX_PLAYLISTS/MAX_ITEMS_PER_PLAYLIST (the store + // is the real enforcement point; this just avoids materializing + // rows it would drop anyway). + db.rawQuery("SELECT uid, name FROM playlists LIMIT 100", null).use { c -> while (c.moveToNext()) { val uid = c.getLong(0) val name = c.getString(1) ?: "Untitled" @@ -338,7 +341,7 @@ object SettingsImport { JOIN streams s ON s.uid = j.stream_id WHERE j.playlist_id = ? ORDER BY j.join_index - LIMIT 5000 + LIMIT 1000 """.trimIndent(), arrayOf(uid.toString()), ).use { c -> diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt index dbc8d8e5b..cb67041c4 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/detail/VideoDetailViewModel.kt @@ -197,18 +197,24 @@ class VideoDetailViewModel : ViewModel() { rydDeferred.await() to sbDeferred.await() } - val related = info.related.map { r -> - StreamItem( - url = r.url, - title = r.title.ifBlank { "(no title)" }, - uploader = r.uploader, - uploaderUrl = r.uploaderUrl, - thumbnail = r.thumbnail, - durationSeconds = r.durationSeconds, - viewCount = r.viewCount, - uploadDateRelative = r.uploadDateRelative, - ) - } + // Dedup by url: the related list is keyed "rel:"+url in the + // LazyColumn, so a duplicate would hard-crash Compose. Empty + // today (core returns no related), but arm the path now so it + // can't regress the moment core starts populating it. + val related = info.related + .distinctBy { it.url } + .map { r -> + StreamItem( + url = r.url, + title = r.title.ifBlank { "(no title)" }, + uploader = r.uploader, + uploaderUrl = r.uploaderUrl, + thumbnail = r.thumbnail, + durationSeconds = r.durationSeconds, + viewCount = r.viewCount, + uploadDateRelative = r.uploadDateRelative, + ) + } // More from this channel via strawcore.channelInfo — one // Rust round-trip returns the channel's Videos tab pre-mapped. @@ -262,6 +268,13 @@ class VideoDetailViewModel : ViewModel() { subscriberCount = ch.subscriberCount, videos = ch.videos .filter { it.url != streamUrl } + // A repeated videoId within the first 20 + // channel-tab entries would produce a + // duplicate LazyColumn key ("mfc:"+url) and + // hard-crash Compose on opening the video. + // Core doesn't dedup channel videos, so gate + // it here (ChannelViewModel does the same). + .distinctBy { it.url } .take(20) .map { v -> StreamItem( diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt new file mode 100644 index 000000000..5de33c408 --- /dev/null +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/diag/LogShipper.kt @@ -0,0 +1,230 @@ +/* + * SPDX-FileCopyrightText: 2026 Sulkta + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Dogfood log shipping — POST a scrubbed logcat window to Kayos's + * ingest service (logs.sulkta.com). Two triggers: + * + * * crash — a default-UncaughtExceptionHandler wrapper captures the + * log ring + the throwable's stacktrace, attempts one + * hard-capped best-effort send, then chains to the + * previously-installed handler so the process still dies + * (and Android still shows its crash dialog) normally. + * * manual — the Settings → Diagnostics "Send logs to Kayos" button. + * + * PRIVACY: every line — including the crash stacktrace — goes through + * LogDump.scrubLine before leaving the device; the server re-scrubs, + * but the phone is the primary gate. device_id is a per-install random + * UUID (no ANDROID_ID / IMEI / hardware fingerprint). And the whole + * feature only exists in builds where BuildConfig.STRAW_LOGS_TOKEN was + * injected (see strawApp/build.gradle.kts) — for everyone else + * `enabled` is a compile-time false, so R8 strips the network path and + * a contributor build cannot phone home. + * + * DELIVERY: strictly best-effort. Any failure (offline, 401/413/429, + * timeout) is swallowed; nothing here may ever crash or block the app. + */ + +package com.sulkta.straw.feature.diag + +import android.content.Context +import com.sulkta.straw.BuildConfig +import com.sulkta.straw.util.LogDump +import com.sulkta.straw.util.runCatchingCancellable +import java.io.PrintWriter +import java.io.StringWriter +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import java.util.TimeZone +import java.util.UUID +import java.util.concurrent.TimeUnit +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody + +object LogShipper { + + /** Not a secret — only the bearer token is. */ + private const val INGEST_URL = "https://logs.sulkta.com/ingest" + + /** Server caps: 10k lines / 1 MB body. Stay well under both. */ + private const val MAX_LINES = 2_000 + private const val MAX_BODY_BYTES = 700_000 + + /** Hard cap on how long the crash path may delay process death. */ + private const val CRASH_JOIN_MS = 4_000L + + private const val PREFS = "straw_diag" + private const val KEY_DEVICE_ID = "device_id" + + /** + * Compile-time gate: builds without an injected token have no + * shipping at all — no network, no crash handler, no Settings row. + */ + val enabled: Boolean + get() = BuildConfig.STRAW_LOGS_TOKEN.isNotEmpty() + + // Short timeouts — this is telemetry, not payload. callTimeout is + // the absolute ceiling on the whole request so nothing here can + // hold a WorkManager-style wakelock pattern open. + private val http: OkHttpClient by lazy { + OkHttpClient.Builder() + .connectTimeout(5, TimeUnit.SECONDS) + .writeTimeout(5, TimeUnit.SECONDS) + .readTimeout(5, TimeUnit.SECONDS) + .callTimeout(10, TimeUnit.SECONDS) + .build() + } + + @Serializable + private data class Envelope( + @SerialName("device_id") val deviceId: String, + @SerialName("app_version") val appVersion: String, + @SerialName("sent_at") val sentAt: String, + val reason: String, + val lines: List, + ) + + /** + * Stable per-install id: `straw-`, minted once and + * persisted. Deliberately NOT ANDROID_ID/IMEI — it identifies an + * install for log correlation, never the hardware or the person. + * commit() (not apply()) because both call paths are already on a + * background thread and the crash path dies right after — an + * async write would lose the id and mint a new one per crash. + */ + private fun deviceId(context: Context): String { + val sp = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + sp.getString(KEY_DEVICE_ID, null)?.let { return it } + val id = "straw-${UUID.randomUUID()}" + sp.edit().putString(KEY_DEVICE_ID, id).commit() + return id + } + + private fun isoNowUtc(): String = + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + .apply { timeZone = TimeZone.getTimeZone("UTC") } + .format(Date()) + + /** + * Manual trigger (Settings). Captures + scrubs the log ring and + * POSTs it. Returns true iff the server 2xx'd — the UI shows a + * toast either way; failures never throw. + */ + suspend fun send(context: Context, reason: String): Boolean = + withContext(Dispatchers.IO) { + if (!enabled) return@withContext false + runCatchingCancellable { + postLines( + context = context, + reason = reason, + lines = LogDump.captureScrubbedLinesBlocking(MAX_LINES), + ) + }.getOrDefault(false) + } + + /** + * Wrap the process-default uncaught-exception handler. Ship-then- + * chain: whatever happens in our path (including a throw), the + * previous handler ALWAYS runs, so the app still crashes/reports + * exactly as before. No-op without a token. + */ + fun installCrashHandler(context: Context) { + if (!enabled) return + val appContext = context.applicationContext + val previous = Thread.getDefaultUncaughtExceptionHandler() + Thread.setDefaultUncaughtExceptionHandler { thread, throwable -> + try { + shipCrash(appContext, throwable) + } catch (_: Throwable) { + // Never let diagnostics mask the real crash. + } + previous?.uncaughtException(thread, throwable) + } + } + + @Volatile + private var crashShipStarted = false + + /** + * Crash-path send. The crashing thread may be main (network there + * throws NetworkOnMainThreadException) and is about to die either + * way, so the capture+POST runs on a dedicated worker thread and + * we join with a hard cap — the death is delayed at most + * [CRASH_JOIN_MS], never hung. + */ + private fun shipCrash(context: Context, throwable: Throwable) { + // One attempt per process — also breaks recursion if the ship + // path itself crashes on another thread mid-flight. + if (crashShipStarted) return + crashShipStarted = true + val worker = Thread { + try { + val trace = StringWriter().also { + throwable.printStackTrace(PrintWriter(it)) + }.toString() + .split('\n') + .take(400) + .map(LogDump::scrubLine) + val ring = LogDump.captureScrubbedLinesBlocking(MAX_LINES) + postLines( + context = context, + reason = "crash", + lines = ring + "--- uncaught exception ---" + trace, + ) + } catch (_: Throwable) { + // Best-effort only. + } + } + worker.name = "straw-crash-ship" + worker.isDaemon = true + worker.start() + try { + worker.join(CRASH_JOIN_MS) + } catch (_: InterruptedException) { + Thread.currentThread().interrupt() + } + } + + /** + * Build the envelope and POST. `lines` MUST already be scrubbed + * (both call paths run everything through LogDump.scrubLine). + * Trims oldest-first to stay under the server's 1 MB cap — one + * bounded request, no chunking needed at our line cap. + */ + private fun postLines(context: Context, reason: String, lines: List): Boolean { + var budget = MAX_BODY_BYTES + val kept = ArrayList(lines.size) + for (i in lines.indices.reversed()) { + // +8 ≈ JSON quotes/comma + escaping slack per line. + val cost = lines[i].toByteArray(Charsets.UTF_8).size + 8 + if (cost > budget) break + budget -= cost + kept.add(lines[i]) + } + kept.reverse() + val envelope = Envelope( + deviceId = deviceId(context), + appVersion = "${BuildConfig.VERSION_NAME} (${BuildConfig.VERSION_CODE})", + sentAt = isoNowUtc(), + reason = reason, + lines = kept, + ) + val body = Json.encodeToString(envelope) + .toRequestBody("application/json; charset=utf-8".toMediaType()) + val request = Request.Builder() + .url(INGEST_URL) + .header("Authorization", "Bearer ${BuildConfig.STRAW_LOGS_TOKEN}") + .post(body) + .build() + return http.newCall(request).execute().use { it.isSuccessful } + } +} diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/NowPlaying.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/NowPlaying.kt index a1f8bb619..be7b6e5c9 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/NowPlaying.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/NowPlaying.kt @@ -75,6 +75,26 @@ object NowPlaying { } } + /** + * Attach richer data (e.g. late-arriving SponsorBlock segments) to the + * currently-playing item ONLY if it is still `item`'s streamUrl. Unlike + * [claim], this NEVER replaces a *different* current — so an async fetch + * that resolves after the user skipped to another video can't hijack + * NowPlaying back to the old one (which would apply stale segments, + * flip the collapsed-follow video, and pollute the resume store). No-op + * if playback has moved on. Race-free: only ever CAS-swaps a same-url + * item, so a concurrent writer flipping `current` to a new url makes the + * next loop's guard bail instead of clobbering it. + */ + fun refreshIfCurrent(item: NowPlayingItem) { + while (true) { + val cur = _current.value ?: return + if (cur.streamUrl != item.streamUrl) return + if (cur == item) return + if (_current.compareAndSet(cur, item)) return + } + } + fun clear() { _current.value = null } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt index 8bd570751..3df7e4ff1 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/player/PlaybackService.kt @@ -258,7 +258,12 @@ class PlaybackService : MediaSessionService() { SponsorBlockClient.fetch(videoId, cats) } if (segments.isNotEmpty()) { - NowPlaying.claim(item.copy(segments = segments)) + // Staleness fence: this SB fetch is a network RTT during + // which the player may have skipped to another track. + // refreshIfCurrent only attaches the segments if `item` is + // STILL the playing video — it will not hijack NowPlaying + // back to this (now-previous) one the way claim() would. + NowPlaying.refreshIfCurrent(item.copy(segments = segments)) } } } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt index 6c96d72d0..35dee1e58 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt @@ -66,6 +66,7 @@ import com.sulkta.straw.data.Settings import com.sulkta.straw.data.ThemeMode import com.sulkta.straw.feature.dataimport.ImportResult import com.sulkta.straw.feature.dataimport.SettingsImport +import com.sulkta.straw.feature.diag.LogShipper import com.sulkta.straw.feature.feed.SubscriptionFeedViewModel import com.sulkta.straw.feature.search.SearchViewModel import com.sulkta.straw.util.LogDump @@ -774,6 +775,39 @@ fun SettingsScreen() { Text(if (logDumping) "Exporting…" else "Export logs…") } + // Dogfood log shipping — only rendered in builds with an + // injected ingest token (LogShipper.enabled is compile-time + // false otherwise, so contributor builds show no phantom row). + if (LogShipper.enabled) { + Spacer(modifier = Modifier.height(16.dp)) + Text( + "Or send the same scrubbed log dump straight to Kayos " + + "(logs.sulkta.com) — no share sheet. URLs, video ids, " + + "and anything token-shaped are redacted on-device first.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Spacer(modifier = Modifier.height(12.dp)) + var logSending by remember { mutableStateOf(false) } + OutlinedButton( + enabled = !logSending, + onClick = { + logSending = true + scope.launch { + val ok = LogShipper.send(context, reason = "manual") + logSending = false + Toast.makeText( + context, + if (ok) "Logs sent — thanks!" else "Couldn't reach the log server", + Toast.LENGTH_LONG, + ).show() + } + }, + ) { + Text(if (logSending) "Sending…" else "Send logs to Kayos") + } + } + Spacer(modifier = Modifier.height(32.dp)) Text( "Import from NewPipe / Tubular", diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/AppUpdateClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/AppUpdateClient.kt index 2800500cf..06160d238 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/AppUpdateClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/AppUpdateClient.kt @@ -21,12 +21,10 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json -import okhttp3.CertificatePinner import okhttp3.OkHttpClient import okhttp3.Request import java.util.concurrent.TimeUnit -private const val INDEX_HOST = "fdroid.sulkta.com" private const val INDEX_URL = "https://fdroid.sulkta.com/fdroid/repo/index-v2.json" private const val REPO_BASE = "https://fdroid.sulkta.com/fdroid/repo" @@ -46,38 +44,56 @@ private val APK_NAME_RE = Regex("""^/[A-Za-z0-9._-]+\.apk$""") */ private const val MAX_PLAUSIBLE_VC = 10_000_000L +/** + * Hard cap on the index body we'll buffer. The straw repo's index-v2.json + * is a few KB; 8 MiB is unreachable organically but stops a ballooned or + * hostile body from OOMing the update check. + */ +private const val MAX_INDEX_BYTES = 8L * 1024 * 1024 + data class UpdateInfo( val versionCode: Long, val versionName: String, val apkUrl: String, ) -object AppUpdateClient { - /** - * Pin two Subject-Public-Key-Info SHA-256 hashes against - * fdroid.sulkta.com so an off-tree CA misissue can't ship the - * user an attacker-signed index. - * - * - sha256/8ofd... — current leaf SPKI. Rotates every ~90 days - * with each Let's Encrypt renewal; an app update before the - * next rotation refreshes this pin. - * - sha256/y7xV... — Let's Encrypt E7 intermediate SPKI. Stable - * for years; serves as the rotation-safety pin while we push - * a new leaf hash. - * - * When the leaf pin no longer matches (post-rotation), OkHttp - * still accepts the chain because the E7 intermediate pin - * matches. The next app release rolls the leaf forward. - */ - private val pinner: CertificatePinner = CertificatePinner.Builder() - .add(INDEX_HOST, "sha256/8ofdiPS6TAiUx9zb2O7Qa9IKZQ3D2i+18teKCrz/MqA=") - .add(INDEX_HOST, "sha256/y7xVm0TVJNahMr2sZydE2jQH8SquXV9yLF9seROHHHU=") - .build() +/** + * Outcome of an index check. Distinguishes "couldn't reach/parse the index" + * (indeterminate — must NOT be reported as up-to-date) from "reached it and + * there is genuinely nothing newer". + */ +sealed interface UpdateResult { + /** Reached + parsed the highest published entry for this package. */ + data class Found(val info: UpdateInfo) : UpdateResult + /** Reached + parsed, but no usable published entry exists. */ + object None : UpdateResult + + /** Could not reach or parse the index — retry without lying "up to date". */ + object Failed : UpdateResult +} + +object AppUpdateClient { + // No certificate pinning — deliberately. We used to pin the + // fdroid.sulkta.com leaf SPKI plus the Let's Encrypt "E7" intermediate, + // but that inverted the risk: the leaf rotates on every ~90-day certbot + // renewal and LE rotates issuing intermediates from a pool (and + // explicitly documents "don't pin our intermediates"), so a routine + // renewal was guaranteed to miss both pins — silently bricking the ONLY + // in-app update path, with the fix reachable only THROUGH the channel the + // break just killed. The threat pinning defended (a CA misissue serving a + // forged index) is already backstopped: a swapped APK cannot install over + // Straw because the Android package installer verifies it against our + // signing key (bb9ca96b…), and the index only names an APK URL. Default + // system-CA validation still blocks a plain MITM. So we drop the + // self-brick and keep the real integrity control (the signature gate). private val http: OkHttpClient = OkHttpClient.Builder() .connectTimeout(15, TimeUnit.SECONDS) .readTimeout(15, TimeUnit.SECONDS) - .certificatePinner(pinner) + // Absolute ceiling on the whole call (connect + write + read + + // redirects) so a slow-drip server can't keep the request — and the + // WorkManager job behind it — alive indefinitely. + .callTimeout(30, TimeUnit.SECONDS) .build() private val json = Json { ignoreUnknownKeys = true } @@ -86,19 +102,27 @@ object AppUpdateClient { * for THIS app's package. Returns null on network/parse failure (the * caller treats null as "no update available, try again later"). */ - suspend fun fetchLatest(): UpdateInfo? = withContext(Dispatchers.IO) { + suspend fun fetchLatest(): UpdateResult = withContext(Dispatchers.IO) { runCatchingCancellable { val req = Request.Builder().url(INDEX_URL).build() val raw = http.newCall(req).execute().use { resp -> - if (!resp.isSuccessful) return@runCatchingCancellable null + if (!resp.isSuccessful) return@runCatchingCancellable UpdateResult.Failed + // Bounded read: request() buffers up to the cap without + // consuming it, so a within-cap body still string()s in full, + // but an attacker-/misconfig-ballooned index is refused + // instead of OOMing the process. + if (resp.body.source().request(MAX_INDEX_BYTES + 1)) { + strawLogW("StrawUpdate") { "index exceeds $MAX_INDEX_BYTES-byte cap — refusing" } + return@runCatchingCancellable UpdateResult.Failed + } resp.body.string() } val index = json.decodeFromString(raw) val pkg = index.packages[BuildConfig.APPLICATION_ID] - ?: return@runCatchingCancellable null + ?: return@runCatchingCancellable UpdateResult.None val best = pkg.versions.values .maxByOrNull { it.manifest.versionCode } - ?: return@runCatchingCancellable null + ?: return@runCatchingCancellable UpdateResult.None // Reject implausible versionCodes outright — see // MAX_PLAUSIBLE_VC. if (best.manifest.versionCode <= 0 || @@ -106,7 +130,7 @@ object AppUpdateClient { strawLogW("StrawUpdate") { "rejecting implausible versionCode=${best.manifest.versionCode}" } - return@runCatchingCancellable null + return@runCatchingCancellable UpdateResult.None } // Strict APK-basename match before we hand this off to // ACTION_VIEW. Anything else gets logged + dropped. @@ -115,14 +139,16 @@ object AppUpdateClient { strawLogW("StrawUpdate") { "rejecting unsafe file.name=${fileName.take(80)}" } - return@runCatchingCancellable null + return@runCatchingCancellable UpdateResult.None } - UpdateInfo( - versionCode = best.manifest.versionCode, - versionName = best.manifest.versionName.orEmpty(), - apkUrl = "$REPO_BASE$fileName", + UpdateResult.Found( + UpdateInfo( + versionCode = best.manifest.versionCode, + versionName = best.manifest.versionName.orEmpty(), + apkUrl = "$REPO_BASE$fileName", + ) ) - }.getOrNull() + }.getOrElse { UpdateResult.Failed } } } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/UpdateCheckWorker.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/UpdateCheckWorker.kt index 758fdabdc..55ab485fc 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/UpdateCheckWorker.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/update/UpdateCheckWorker.kt @@ -24,6 +24,7 @@ import android.app.PendingIntent import android.content.Context import android.content.Intent import android.net.Uri +import android.os.Build import androidx.core.app.NotificationCompat import androidx.work.CoroutineWorker import androidx.work.WorkerParameters @@ -36,23 +37,37 @@ import com.sulkta.straw.util.strawLogI * Touched by both the scheduled worker AND the "Check now" Settings * button so behavior stays identical regardless of trigger. */ -suspend fun runUpdateCheck(context: Context): UpdateInfo? { - val info = AppUpdateClient.fetchLatest() - Settings.get().setLastUpdateCheck(System.currentTimeMillis()) - if (info == null) { - strawLogI("update", "check: network/parse failure, will retry") - return null +suspend fun runUpdateCheck(context: Context): UpdateInfo? = + when (val result = AppUpdateClient.fetchLatest()) { + // Indeterminate — do NOT advance the last-check timestamp, or a + // permanently-failing checker would read "checked just now / up to + // date" while it's actually blind. Retry on the next tick. + is UpdateResult.Failed -> { + strawLogI("update", "check: network/parse failure, will retry (timestamp not advanced)") + null + } + // Reached the repo; it has nothing usable for this package. + is UpdateResult.None -> { + Settings.get().setLastUpdateCheck(System.currentTimeMillis()) + Settings.get().setLatestKnownVersion(0L, "") + strawLogI("update", "check: no published build for this package") + null + } + is UpdateResult.Found -> { + val info = result.info + Settings.get().setLastUpdateCheck(System.currentTimeMillis()) + if (info.versionCode <= BuildConfig.VERSION_CODE) { + strawLogI("update", "check: up to date (latest=${info.versionCode})") + Settings.get().setLatestKnownVersion(0L, "") + null + } else { + strawLogI("update", "check: ${BuildConfig.VERSION_CODE} → ${info.versionCode} available") + Settings.get().setLatestKnownVersion(info.versionCode, info.versionName) + postUpdateNotification(context, info) + info + } + } } - if (info.versionCode <= BuildConfig.VERSION_CODE) { - strawLogI("update", "check: up to date (latest=${info.versionCode})") - Settings.get().setLatestKnownVersion(0L, "") - return null - } - strawLogI("update", "check: ${BuildConfig.VERSION_CODE} → ${info.versionCode} available") - Settings.get().setLatestKnownVersion(info.versionCode, info.versionName) - postUpdateNotification(context, info) - return info -} class UpdateCheckWorker( context: Context, @@ -74,14 +89,19 @@ private const val NOTIF_ID = 23 private fun postUpdateNotification(context: Context, info: UpdateInfo) { val nm = context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager - val channel = NotificationChannel( - NOTIF_CHANNEL_ID, - "Straw updates", - NotificationManager.IMPORTANCE_DEFAULT, - ).apply { - description = "Notifies when a newer Straw build is on fdroid.sulkta.com." + // NotificationChannel is API 26+; minSdk is 24, so guard it or a 7.x + // device crashes (NoClassDefFoundError) the moment an update is found. + // Pre-O, NotificationCompat.Builder ignores the channel id and posts fine. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val channel = NotificationChannel( + NOTIF_CHANNEL_ID, + "Straw updates", + NotificationManager.IMPORTANCE_DEFAULT, + ).apply { + description = "Notifies when a newer Straw build is on fdroid.sulkta.com." + } + nm.createNotificationChannel(channel) } - nm.createNotificationChannel(channel) // ACTION_VIEW on the APK URL — Chrome / system browser fetches it // via DownloadManager and the user taps it to install. No @@ -102,6 +122,10 @@ private fun postUpdateNotification(context: Context, info: UpdateInfo) { .setContentTitle("Straw $name available") .setContentText("Tap to download from fdroid.sulkta.com.") .setAutoCancel(true) + // Only sound/vibrate the first time we surface a given version — the + // check re-runs on every cadence tick and would otherwise re-nag for + // the same pending update until the user installs it. + .setOnlyAlertOnce(true) .setContentIntent(pending) .build() nm.notify(NOTIF_ID, notif) diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt index 400127de3..e47fe8d13 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/RydClient.kt @@ -10,6 +10,7 @@ package com.sulkta.straw.net +import com.sulkta.straw.util.runCatchingCancellable import kotlinx.serialization.Serializable @Serializable @@ -23,13 +24,18 @@ object RydClient { /** Suspends on the Rust async runtime; call from a coroutine. Returns * null on any failure (transport / non-2xx / bad JSON), same contract * the old blocking client had. Only likes/dislikes are carried — RYD's - * rating/viewCount were dead, redundant overlay data (audit #2 L-9). */ + * rating/viewCount were dead, redundant overlay data (audit #2 L-9). + * The uniffi call is wrapped: a core panic now unwinds to a UniFFI + * InternalException (panic=unwind), which this shim's "null on failure" + * contract must swallow rather than crash the detail screen. */ suspend fun fetch(videoId: String): RydVotes? = - uniffi.strawcore.fetchRydVotes(videoId)?.let { v -> - RydVotes( - id = v.id, - likes = v.likes, - dislikes = v.dislikes, - ) - } + runCatchingCancellable { + uniffi.strawcore.fetchRydVotes(videoId)?.let { v -> + RydVotes( + id = v.id, + likes = v.likes, + dislikes = v.dislikes, + ) + } + }.getOrNull() } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt index f15a6ef6d..662db3eea 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/net/SponsorBlockClient.kt @@ -12,6 +12,7 @@ package com.sulkta.straw.net +import com.sulkta.straw.util.runCatchingCancellable import kotlinx.serialization.Serializable @Serializable @@ -28,17 +29,22 @@ data class SbSegment( object SponsorBlockClient { /** Suspends on the Rust async runtime; call from a coroutine. Returns * an empty list on any failure, same contract the old blocking client - * had. `categories` are the SB category keys to request. */ + * had. `categories` are the SB category keys to request. The uniffi + * call is wrapped so a core panic (now unwinding to a UniFFI + * InternalException under panic=unwind) is swallowed to an empty list + * rather than crashing video-detail load. */ suspend fun fetch( videoId: String, categories: List = listOf("sponsor"), ): List = - uniffi.strawcore.fetchSponsorSegments(videoId, categories).map { s -> - SbSegment( - UUID = s.uuid, - category = s.category, - segment = listOf(s.startSec, s.endSec), - actionType = s.actionType, - ) - } + runCatchingCancellable { + uniffi.strawcore.fetchSponsorSegments(videoId, categories).map { s -> + SbSegment( + UUID = s.uuid, + category = s.category, + segment = listOf(s.startSec, s.endSec), + actionType = s.actionType, + ) + } + }.getOrDefault(emptyList()) } diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt b/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt index 159e7b8aa..ec922c4bd 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/util/LogDump.kt @@ -96,10 +96,46 @@ object LogDump { } /** - * Pre-redact known credential-shaped substrings before they hit - * disk. Cheap line-level pass — adversarial-perfect would need a - * URL parser, but the regex approach catches every documented - * leak vector at zero allocation cost. + * Pull recent logcat, scrub every line, keep only the newest + * [maxLines]. BLOCKING — callers must already be off the main + * thread (LogShipper runs on Dispatchers.IO; the crash handler + * runs this on its own short-lived worker thread, where a suspend + * fun can't be awaited without an event loop). + * + * Best-effort by design: a logcat exec failure returns whatever + * was read (possibly empty) instead of throwing, so the crash + * shipper can still deliver the stacktrace. + */ + fun captureScrubbedLinesBlocking(maxLines: Int): List { + val ring = ArrayDeque() + runCatching { + val pid = Process.myPid() + val cmd = arrayOf("logcat", "-d", "-v", "threadtime", "--pid=$pid") + val proc = ProcessBuilder(*cmd).redirectErrorStream(true).start() + proc.inputStream.bufferedReader().useLines { lines -> + lines.forEach { line -> + if (ring.size == maxLines) ring.removeFirst() + ring.addLast(line) + } + } + proc.waitFor() + } + // Scrub only the retained window — cheaper than scrubbing lines + // that get dropped, and nothing raw ever leaves this function. + return ring.map(::scrubLine) + } + + /** + * Pre-redact known credential- and identity-shaped substrings + * before they hit disk or the wire. Cheap line-level pass — + * adversarial-perfect would need a URL parser, but the regex + * approach catches every documented leak vector at zero + * allocation cost. + * + * BIAS: over-redaction. These lines leave the device (share sheet + * + LogShipper → logs.sulkta.com), so a false positive costs a + * little debug signal while a false negative leaks what someone + * watched. Anything URL-, token-, id-, or address-shaped goes. * * Public so error-handler call sites (PlayerScreen / VideoDetail * `playbackError`) can scrub Media3's `PlaybackException.message` @@ -109,28 +145,112 @@ object LogDump { */ fun scrubLine(line: String): String { var s = line - // Pre-signed googlevideo URLs: keep host visible, drop path+query. + // Pre-signed googlevideo URLs: keep the host label visible, drop + // host prefix + path + query (session-bound streaming creds). s = GOOGLEVIDEO_URL_RE.replace(s, "https://.googlevideo.com/") + // Credential-shaped headers / key-value pairs, wherever they + // appear: `Authorization: Bearer x`, `cookie: …`, `api_key=…`. + s = BEARER_RE.replace(s, "$1 ") + s = CRED_KV_RE.replace(s, "$1: ") // Long, distinctive token names — match anywhere. s = SIGNED_PARAM_LONG_RE.replace(s, "$1=") // Short single-letter / two-letter tokens — require `[?&]` // immediately before to avoid eating innocent counters. s = SIGNED_PARAM_SHORT_RE.replace(s, "$1$2=") + // ANY remaining URL: keep scheme+host (routing/debug signal, + // not PII), drop path + query — that's where watch?v=…, + // /vi// thumbnails, search terms, etc. live. + s = URL_TAIL_RE.replace(s, "$1/") + // Schemeless YT links pasted into log messages ("youtu.be/xyz"). + s = SCHEMELESS_YT_RE.replace(s, "$1/") + // Emails. + s = EMAIL_RE.replace(s, "") + // YouTube channel + playlist ids (watch behavior). + s = CHANNEL_ID_RE.replace(s, "") + s = PLAYLIST_ID_RE.replace(s, "") + // Bare 11-char video-id-shaped tokens ("dQw4w9WgXcQ") logged + // outside any URL (rustypipe/strawcore do this). Requiring a + // digit/_/- inside the run spares ordinary 11-letter words; + // real ids virtually always contain one. Pure-alpha ids are + // still caught earlier when keyed (v=…) or inside a URL. + s = VIDEO_ID_CANDIDATE_RE.replace(s) { m -> + if (m.value.any { it.isDigit() || it == '_' || it == '-' }) "" else m.value + } + // Long high-entropy runs (hashes, visitor data, unlabeled + // tokens): 20+ [A-Za-z0-9_-] chars containing a digit. + s = LONG_TOKEN_RE.replace(s, "") + // IP addresses (v4 + v6 — v6 patterns are shaped so threadtime + // HH:MM:SS timestamps can never match). + s = IPV4_RE.replace(s, "") + s = IPV6_FULL_RE.replace(s, "") + s = IPV6_COMPRESSED_RE.replace(s, "") return s } private val GOOGLEVIDEO_URL_RE = Regex( """https?://[a-zA-Z0-9.-]*googlevideo\.com/\S+""", ) + // `Authorization: Bearer ` and friends. The token charset is + // the RFC 6750 b64token alphabet. + private val BEARER_RE = Regex( + """\b(bearer)\s+[A-Za-z0-9._~+/=-]{4,}""", + RegexOption.IGNORE_CASE, + ) + // Credential-y names followed by `: value` or `= value` (optional + // closing quote for JSON `"token":"…"` shapes). Values may be + // quoted strings or a bare token. + private val CRED_KV_RE = Regex( + """\b(authorization|proxy-authorization|cookie|set-cookie|x-goog-[a-z\-]+|x-youtube-[a-z\-]+|api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|client[-_]?secret|secret|token|auth|session[-_]?id|passw(?:or)?d|pwd|mnemonic|private[-_]?key)"?\s*[:=]\s*("[^"]*"|\S+)""", + RegexOption.IGNORE_CASE, + ) // Long tokens are unique enough to match anywhere. Short tokens - // (n, mn, ms, mo, pl, ip, ei) require `[?&]` immediately before - // so we don't redact innocuous `n=42` counters from other libs. + // (n, mn, ms, … v, id, q) require `[?&]` immediately before so we + // don't redact innocuous `n=42` counters from other libs. `v`, + // `id`, `list`, `q`, `t` are the YouTube identity/search params. private val SIGNED_PARAM_LONG_RE = Regex( - """\b(signature|sparams|lsig|cpn|expire|pot|sig|key)=([^&\s"']+)""", + """\b(signature|sparams|lsig|cpn|expire|pot|sig|key|videoId|video_id|docid|search_query|query)=([^&\s"']+)""", RegexOption.IGNORE_CASE, ) private val SIGNED_PARAM_SHORT_RE = Regex( - """([?&])(n|mn|ms|mo|pl|ip|ei)=([^&\s"']+)""", + """([?&])(n|mn|ms|mo|pl|ip|ei|v|id|list|q|t)=([^&\s"']+)""", RegexOption.IGNORE_CASE, ) + // Any http(s) URL: capture scheme + host(+port), scrub the rest. + // Host charset can't contain `<`, so the googlevideo replacement + // above is never re-matched. + private val URL_TAIL_RE = Regex( + """(https?://[A-Za-z0-9.-]+(?::\d+)?)[/?#][^\s"'<>]*""", + ) + // youtu.be/… + youtube.com/… without a scheme. The lookbehind + // skips hosts already handled as part of a full URL (preceded by + // `/` or a subdomain dot). + private val SCHEMELESS_YT_RE = Regex( + """(?]*""", + ) + private val EMAIL_RE = Regex( + """[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}""", + ) + private val CHANNEL_ID_RE = Regex( + """(?