vc=24: NewPipe/Tubular settings import

Settings screen now has an Import section. Picks a NewPipeData-*.zip or TubularData-*.zip via the SAF, extracts newpipe.db + preferences.json, and walks the Room SQLite schema (subscriptions / playlists / playlist_stream_join / streams / search_history / stream_history / stream_state). YouTube only — service_id=0 filter drops SoundCloud/PeerTube/etc.

Imported on smoke: 26 subs, 1 playlist with 10 items, 50/11402 watch history (capped by HistoryStore MAX_WATCHES), 20/2242 searches (MAX_SEARCHES), 8 settings keys (SponsorBlock category toggles + default resolution). Resume positions counted (10918) but not yet persisted — Straw has no resume-store yet.

buildSrc/src/main/kotlin/ProjectConfig.kt

@@ -55,6 +55,6 @@ const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app"
 // vc=19 / 0.1.0-AE — rust pipeline cutover. Extraction via
 // strawcore-core (Sulkta-OSS/strawcore) via the UniFFI wrapper; no
 // NewPipeExtractor in the runtime path.
-const val STRAW_VERSION_CODE = 23
+const val STRAW_VERSION_CODE = 24
-const val STRAW_VERSION_NAME = "0.1.0-AI"
+const val STRAW_VERSION_NAME = "0.1.0-AJ"
 const val STRAW_APPLICATION_ID = "com.sulkta.straw"

strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt

@@ -0,0 +1,419 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Sulkta
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ * NewPipe / Tubular export importer.
+ *
+ * The user picks an exported `.zip` (NewPipe writes it as
+ * `NewPipeData-<date>.zip`, Tubular as `TubularData-<date>.zip`).
+ * Inside:
+ *   - newpipe.db        Room SQLite (subscriptions, playlists, history…)
+ *   - preferences.json  flat key/value of all user settings
+ *   - newpipe.settings  superseded XML form of preferences (we ignore)
+ *
+ * We populate Straw's existing stores (Subscriptions, Playlists, History,
+ * Settings) — filtering to service_id=0 (YouTube). Other services
+ * (SoundCloud / PeerTube / …) are silently dropped — we don't support
+ * them and a mixed import would surprise the user later.
+ *
+ * Resume positions (NewPipe `stream_state` table) are read but
+ * intentionally not persisted yet — Straw has no resume-positions
+ * store. Counted in [ImportResult.resumePositionsSeen] so the user
+ * knows the data was present even if dropped.
+ */
+
+package com.sulkta.straw.feature.dataimport
+
+import android.content.Context
+import android.database.sqlite.SQLiteDatabase
+import android.net.Uri
+import com.sulkta.straw.data.ChannelRef
+import com.sulkta.straw.data.History
+import com.sulkta.straw.data.PlaylistItem
+import com.sulkta.straw.data.Playlists
+import com.sulkta.straw.data.SbCategory
+import com.sulkta.straw.data.Settings
+import com.sulkta.straw.data.Subscriptions
+import com.sulkta.straw.data.WatchHistoryItem
+import java.io.File
+import java.util.zip.ZipInputStream
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.boolean
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.jsonPrimitive
+
+data class ImportResult(
+    val subscriptionsAdded: Int,
+    val subscriptionsSkippedNonYt: Int,
+    val playlistsAdded: Int,
+    val playlistItemsAdded: Int,
+    val searchHistoryAdded: Int,
+    val searchHistoryAvailable: Int,
+    val watchHistoryAdded: Int,
+    val watchHistoryAvailable: Int,
+    val resumePositionsSeen: Int,
+    val settingsApplied: Int,
+    val warnings: List<String>,
+) {
+    fun summary(): String = buildString {
+        append("Imported ")
+        append(subscriptionsAdded)
+        append(" subs")
+        if (subscriptionsSkippedNonYt > 0) {
+            append(" (skipped ")
+            append(subscriptionsSkippedNonYt)
+            append(" non-YouTube)")
+        }
+        append(", ")
+        append(playlistsAdded)
+        append(" playlist")
+        if (playlistsAdded != 1) append("s")
+        append(" (")
+        append(playlistItemsAdded)
+        append(" videos), ")
+        append(watchHistoryAdded)
+        append("/")
+        append(watchHistoryAvailable)
+        append(" watch history, ")
+        append(searchHistoryAdded)
+        append("/")
+        append(searchHistoryAvailable)
+        append(" searches, ")
+        append(settingsApplied)
+        append(" settings.")
+        if (resumePositionsSeen > 0) {
+            append(" Resume positions (")
+            append(resumePositionsSeen)
+            append(") not yet supported — dropped.")
+        }
+        if (warnings.isNotEmpty()) {
+            append("\n\nWarnings:\n")
+            warnings.forEach { append("• "); append(it); append("\n") }
+        }
+    }
+}
+
+object SettingsImport {
+
+    // YouTube only — Straw doesn't extract from other services.
+    private const val YT_SERVICE_ID = 0
+
+    suspend fun run(context: Context, zipUri: Uri): Result<ImportResult> =
+        withContext(Dispatchers.IO) {
+            runCatching { runInner(context, zipUri) }
+        }
+
+    private fun runInner(context: Context, zipUri: Uri): ImportResult {
+        val warnings = mutableListOf<String>()
+        val workDir = File(context.cacheDir, "newpipe-import-${System.currentTimeMillis()}")
+        workDir.mkdirs()
+        try {
+            val (dbFile, prefsJson) = extractZip(context, zipUri, workDir, warnings)
+
+            val subsResult = if (dbFile != null) importSubscriptions(dbFile) else SubsResult(0, 0)
+            val plResult = if (dbFile != null) importPlaylists(dbFile) else PlResult(0, 0)
+            val histResult = if (dbFile != null) importHistory(dbFile) else HistResult(0, 0, 0, 0, 0)
+            val settingsResult = if (prefsJson != null) importSettings(prefsJson) else 0
+
+            return ImportResult(
+                subscriptionsAdded = subsResult.added,
+                subscriptionsSkippedNonYt = subsResult.skipped,
+                playlistsAdded = plResult.playlists,
+                playlistItemsAdded = plResult.items,
+                searchHistoryAdded = histResult.searches,
+                searchHistoryAvailable = histResult.searchesAvailable,
+                watchHistoryAdded = histResult.watchesAdded,
+                watchHistoryAvailable = histResult.watchesAvailable,
+                resumePositionsSeen = histResult.resumePositions,
+                settingsApplied = settingsResult,
+                warnings = warnings,
+            )
+        } finally {
+            workDir.deleteRecursively()
+        }
+    }
+
+    private fun extractZip(
+        context: Context,
+        zipUri: Uri,
+        workDir: File,
+        warnings: MutableList<String>,
+    ): Pair<File?, JsonObject?> {
+        var dbFile: File? = null
+        var prefs: JsonObject? = null
+        context.contentResolver.openInputStream(zipUri)?.use { input ->
+            ZipInputStream(input).use { zip ->
+                while (true) {
+                    val entry = zip.nextEntry ?: break
+                    when (entry.name) {
+                        "newpipe.db" -> {
+                            val out = File(workDir, "newpipe.db")
+                            out.outputStream().use { os ->
+                                zip.copyTo(os, bufferSize = 64 * 1024)
+                            }
+                            dbFile = out
+                        }
+                        "preferences.json" -> {
+                            val bytes = zip.readBytes()
+                            prefs = runCatching {
+                                Json.parseToJsonElement(bytes.decodeToString()) as? JsonObject
+                            }.getOrNull()
+                            if (prefs == null) warnings += "preferences.json present but unparseable"
+                        }
+                        // newpipe.settings is the legacy XML form; preferences.json
+                        // supersedes it in every modern export. Skip.
+                        else -> { /* ignore other entries */ }
+                    }
+                    zip.closeEntry()
+                }
+            }
+        } ?: error("Could not open the selected file")
+        if (dbFile == null) warnings += "newpipe.db not found in archive — most data skipped"
+        if (prefs == null) warnings += "preferences.json not found — settings not migrated"
+        return dbFile to prefs
+    }
+
+    private data class SubsResult(val added: Int, val skipped: Int)
+    private fun importSubscriptions(dbFile: File): SubsResult {
+        val store = Subscriptions.get()
+        var added = 0
+        var skipped = 0
+        openDb(dbFile).use { db ->
+            db.rawQuery(
+                "SELECT url, name, avatar_url, service_id FROM subscriptions",
+                null,
+            ).use { c ->
+                while (c.moveToNext()) {
+                    val serviceId = c.getInt(3)
+                    if (serviceId != YT_SERVICE_ID) {
+                        skipped++
+                        continue
+                    }
+                    val url = c.getString(0) ?: continue
+                    val name = c.getString(1) ?: continue
+                    val avatar = c.getString(2)
+                    if (!store.isSubscribed(url)) {
+                        store.toggle(ChannelRef(url = url, name = name, avatar = avatar))
+                        added++
+                    }
+                }
+            }
+        }
+        return SubsResult(added, skipped)
+    }
+
+    private data class PlResult(val playlists: Int, val items: Int)
+    private fun importPlaylists(dbFile: File): PlResult {
+        val store = Playlists.get()
+        var playlistsAdded = 0
+        var itemsAdded = 0
+        openDb(dbFile).use { db ->
+            val playlistRows = mutableListOf<Pair<Long, String>>()
+            db.rawQuery("SELECT uid, name FROM playlists", null).use { c ->
+                while (c.moveToNext()) {
+                    val uid = c.getLong(0)
+                    val name = c.getString(1) ?: "Untitled"
+                    playlistRows += uid to name
+                }
+            }
+            for ((uid, name) in playlistRows) {
+                val items = mutableListOf<PlaylistItem>()
+                db.rawQuery(
+                    """
+                    SELECT s.url, s.title, s.thumbnail_url, s.uploader, s.service_id
+                    FROM playlist_stream_join j
+                    JOIN streams s ON s.uid = j.stream_id
+                    WHERE j.playlist_id = ?
+                    ORDER BY j.join_index
+                    """.trimIndent(),
+                    arrayOf(uid.toString()),
+                ).use { c ->
+                    while (c.moveToNext()) {
+                        if (c.getInt(4) != YT_SERVICE_ID) continue
+                        items += PlaylistItem(
+                            streamUrl = c.getString(0) ?: continue,
+                            title = c.getString(1) ?: "(no title)",
+                            thumbnail = c.getString(2),
+                            uploader = c.getString(3) ?: "",
+                            addedAt = System.currentTimeMillis(),
+                        )
+                    }
+                }
+                if (items.isEmpty()) continue
+                // Use the store's normal create + addItem rather than minting
+                // a Playlist directly — keeps the atomic-update path
+                // consistent with user-driven creates.
+                val created = store.create(name)
+                for (it in items) store.addItem(created.id, it)
+                playlistsAdded++
+                itemsAdded += items.size
+            }
+        }
+        return PlResult(playlistsAdded, itemsAdded)
+    }
+
+    private data class HistResult(
+        val watchesAdded: Int,
+        val watchesAvailable: Int,
+        val searches: Int,
+        val searchesAvailable: Int,
+        val resumePositions: Int,
+    )
+
+    private fun importHistory(dbFile: File): HistResult {
+        val historyStore = History.get()
+        var watchesSeen = 0
+        var watchesAvailable = 0
+        var searchesSeen = 0
+        var resumePositions = 0
+        val searchesBefore = historyStore.searches.value.size
+        val watchesBefore = historyStore.watches.value.size
+        openDb(dbFile).use { db ->
+            // Search history — feed oldest first so the store ends up with
+            // the most-recent on top after its own dedup + take(MAX).
+            db.rawQuery(
+                "SELECT search FROM search_history WHERE service_id=? ORDER BY creation_date ASC",
+                arrayOf(YT_SERVICE_ID.toString()),
+            ).use { c ->
+                while (c.moveToNext()) {
+                    val q = c.getString(0) ?: continue
+                    historyStore.recordSearch(q)
+                    searchesSeen++
+                }
+            }
+
+            // Watch history — newest first via stream_history.access_date,
+            // joined to streams for the metadata we need.
+            // recordWatch caps internally; we just stop counting "added" once
+            // we've replayed Straw's MAX rows. (The store reverses to put
+            // most-recent on top — so we feed it oldest-first to match.)
+            db.rawQuery("SELECT COUNT(*) FROM stream_history", null).use { c ->
+                if (c.moveToNext()) watchesAvailable = c.getInt(0)
+            }
+            db.rawQuery(
+                """
+                SELECT s.url, s.title, s.uploader, s.thumbnail_url, h.access_date, s.service_id
+                FROM stream_history h
+                JOIN streams s ON s.uid = h.stream_id
+                ORDER BY h.access_date ASC
+                """.trimIndent(),
+                null,
+            ).use { c ->
+                while (c.moveToNext()) {
+                    if (c.getInt(5) != YT_SERVICE_ID) continue
+                    val url = c.getString(0) ?: continue
+                    val title = c.getString(1) ?: continue
+                    val uploader = c.getString(2) ?: ""
+                    val thumb = c.getString(3)
+                    val videoId = extractYtVideoId(url) ?: continue
+                    historyStore.recordWatch(
+                        WatchHistoryItem(
+                            url = url,
+                            videoId = videoId,
+                            title = title,
+                            uploader = uploader,
+                            thumbnail = thumb,
+                            watchedAt = c.getLong(4),
+                        ),
+                    )
+                    watchesSeen++
+                }
+            }
+
+            // Resume positions — counted, not stored. Future task hooks into
+            // a ResumePositionsStore.
+            db.rawQuery("SELECT COUNT(*) FROM stream_state", null).use { c ->
+                if (c.moveToNext()) resumePositions = c.getInt(0)
+            }
+        }
+        // Report what actually landed in the store after its dedup + caps.
+        return HistResult(
+            watchesAdded = historyStore.watches.value.size - watchesBefore,
+            watchesAvailable = watchesAvailable.takeIf { it > 0 } ?: watchesSeen,
+            searches = historyStore.searches.value.size - searchesBefore,
+            resumePositions = resumePositions,
+            searchesAvailable = searchesSeen,
+        )
+    }
+
+    private fun importSettings(prefs: JsonObject): Int {
+        val settings = Settings.get()
+        var applied = 0
+
+        // SponsorBlock: master toggle gates the categories. If disabled in
+        // NewPipe, leave Straw's categories alone (they have a non-empty
+        // default). If enabled, sync each category boolean.
+        val sbMaster = prefs.boolOrNull("sponsor_block_enable")
+        if (sbMaster == true) {
+            val targets = mapOf(
+                "sponsor_block_category_sponsor" to SbCategory.Sponsor,
+                "sponsor_block_category_self_promo" to SbCategory.SelfPromo,
+                "sponsor_block_category_intro" to SbCategory.Intro,
+                "sponsor_block_category_outro" to SbCategory.Outro,
+                "sponsor_block_category_interaction" to SbCategory.Interaction,
+                "sponsor_block_category_music" to SbCategory.MusicOfftopic,
+                "sponsor_block_category_filler" to SbCategory.Filler,
+            )
+            val current = settings.sbCategories.value
+            for ((key, cat) in targets) {
+                val want = prefs.boolOrNull(key) ?: continue
+                val have = cat in current
+                if (want != have) settings.toggle(cat)
+                applied++
+            }
+        }
+
+        // Default resolution: NewPipe values like "720p60", "1080p", "Best
+        // resolution". Map down to Straw's discrete ceilings.
+        prefs.stringOrNull("default_resolution")?.let { raw ->
+            val r = parseResolution(raw)
+            if (r != null) {
+                settings.setMaxResolution(r)
+                applied++
+            }
+        }
+
+        return applied
+    }
+
+    private fun parseResolution(raw: String): com.sulkta.straw.data.MaxResolution? {
+        val n = Regex("(\\d+)").find(raw)?.groupValues?.get(1)?.toIntOrNull()
+            ?: return when (raw.lowercase()) {
+                "best resolution", "best", "highest" -> com.sulkta.straw.data.MaxResolution.Auto
+                else -> null
+            }
+        return when {
+            n >= 1080 -> com.sulkta.straw.data.MaxResolution.P1080
+            n >= 720 -> com.sulkta.straw.data.MaxResolution.P720
+            n >= 480 -> com.sulkta.straw.data.MaxResolution.P480
+            n >= 360 -> com.sulkta.straw.data.MaxResolution.P360
+            else -> com.sulkta.straw.data.MaxResolution.P144
+        }
+    }
+
+    private fun openDb(dbFile: File): SQLiteDatabase =
+        SQLiteDatabase.openDatabase(
+            dbFile.absolutePath,
+            /* factory = */ null,
+            SQLiteDatabase.OPEN_READONLY,
+        )
+
+    // YouTube URL patterns we need to parse for the videoId column on
+    // WatchHistoryItem. Cover the watch?v= form (canonical), youtu.be
+    // shortlinks, and embed/. Reject anything we can't parse rather than
+    // inventing IDs.
+    private val YT_ID = Regex(
+        "(?:youtu\\.be/|youtube(?:-nocookie)?\\.com/(?:watch\\?(?:.*&)?v=|embed/|v/|shorts/))([A-Za-z0-9_-]{6,15})",
+    )
+    private fun extractYtVideoId(url: String): String? =
+        YT_ID.find(url)?.groupValues?.get(1)
+
+    private fun JsonObject.boolOrNull(key: String): Boolean? =
+        runCatching { this[key]?.jsonPrimitive?.boolean }.getOrNull()
+
+    private fun JsonObject.stringOrNull(key: String): String? =
+        runCatching { this[key]?.jsonPrimitive?.contentOrNull }.getOrNull()
+}

strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt

@@ -5,41 +5,66 @@
 
 package com.sulkta.straw.feature.settings
 
+import androidx.activity.compose.rememberLauncherForActivityResult
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.compose.foundation.clickable
 import androidx.compose.foundation.layout.Arrangement
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Row
 import androidx.compose.foundation.layout.Spacer
 import androidx.compose.foundation.layout.fillMaxSize
-import androidx.compose.foundation.layout.statusBarsPadding
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.height
 import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.statusBarsPadding
 import androidx.compose.foundation.rememberScrollState
 import androidx.compose.foundation.verticalScroll
+import androidx.compose.material3.AlertDialog
+import androidx.compose.material3.Button
+import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.HorizontalDivider
 import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.OutlinedButton
 import androidx.compose.material3.Switch
 import androidx.compose.material3.Text
-import androidx.compose.foundation.clickable
+import androidx.compose.material3.TextButton
-import androidx.compose.foundation.layout.fillMaxWidth
-import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
+import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.text.font.FontWeight
 import androidx.compose.ui.unit.dp
 import com.sulkta.straw.data.History
 import com.sulkta.straw.data.MaxResolution
 import com.sulkta.straw.data.SbCategory
 import com.sulkta.straw.data.Settings
+import com.sulkta.straw.feature.dataimport.ImportResult
+import com.sulkta.straw.feature.dataimport.SettingsImport
+import kotlinx.coroutines.launch
 
 @Composable
 fun SettingsScreen() {
     val store = Settings.get()
     val cats by store.sbCategories.collectAsState()
+    val context = LocalContext.current
+    val scope = rememberCoroutineScope()
+
+    var importRunning by remember { mutableStateOf(false) }
+    var importResult by remember { mutableStateOf<Result<ImportResult>?>(null) }
+    val pickZip = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
+        if (uri == null) return@rememberLauncherForActivityResult
+        importRunning = true
+        scope.launch {
+            importResult = SettingsImport.run(context, uri)
+            importRunning = false
+        }
+    }
 
     Column(
         modifier = Modifier
@@ -124,6 +149,54 @@ fun SettingsScreen() {
                 Text("Clear searches")
             }
         }
+
+        Spacer(modifier = Modifier.height(32.dp))
+        Text(
+            "Import from NewPipe / Tubular",
+            style = MaterialTheme.typography.titleMedium,
+            fontWeight = FontWeight.SemiBold,
+        )
+        Spacer(modifier = Modifier.height(4.dp))
+        Text(
+            "Pick a NewPipeData-*.zip or TubularData-*.zip — we'll lift " +
+                "your subscriptions, playlists, search history, watch history " +
+                "(capped to 50 most recent), and a curated subset of settings. " +
+                "Other services (SoundCloud, PeerTube) are skipped.",
+            style = MaterialTheme.typography.bodySmall,
+            color = MaterialTheme.colorScheme.onSurfaceVariant,
+        )
+        Spacer(modifier = Modifier.height(12.dp))
+        Button(
+            enabled = !importRunning,
+            onClick = { pickZip.launch(arrayOf("application/zip", "application/octet-stream", "*/*")) },
+        ) {
+            if (importRunning) {
+                CircularProgressIndicator(
+                    modifier = Modifier.height(18.dp).padding(end = 8.dp),
+                    strokeWidth = 2.dp,
+                )
+                Text("Importing…")
+            } else {
+                Text("Pick export file…")
+            }
+        }
+    }
+
+    importResult?.let { res ->
+        AlertDialog(
+            onDismissRequest = { importResult = null },
+            title = { Text(if (res.isSuccess) "Import complete" else "Import failed") },
+            text = {
+                val body = res.fold(
+                    onSuccess = { it.summary() },
+                    onFailure = { it.message ?: it.javaClass.simpleName },
+                )
+                Text(body, style = MaterialTheme.typography.bodyMedium)
+            },
+            confirmButton = {
+                TextButton(onClick = { importResult = null }) { Text("OK") }
+            },
+        )
     }
 }