From 1443bb8ef77aee4346e3dfc2475eb521c747d5b7 Mon Sep 17 00:00:00 2001 From: Kayos Date: Mon, 25 May 2026 16:44:27 +0000 Subject: [PATCH] vc=24: NewPipe/Tubular settings import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings screen now has an Import section. Picks a NewPipeData-*.zip or TubularData-*.zip via the SAF, extracts newpipe.db + preferences.json, and walks the Room SQLite schema (subscriptions / playlists / playlist_stream_join / streams / search_history / stream_history / stream_state). YouTube only — service_id=0 filter drops SoundCloud/PeerTube/etc. Imported on smoke: 26 subs, 1 playlist with 10 items, 50/11402 watch history (capped by HistoryStore MAX_WATCHES), 20/2242 searches (MAX_SEARCHES), 8 settings keys (SponsorBlock category toggles + default resolution). Resume positions counted (10918) but not yet persisted — Straw has no resume-store yet. --- buildSrc/src/main/kotlin/ProjectConfig.kt | 4 +- .../feature/dataimport/SettingsImport.kt | 419 ++++++++++++++++++ .../straw/feature/settings/SettingsScreen.kt | 81 +++- 3 files changed, 498 insertions(+), 6 deletions(-) create mode 100644 strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt diff --git a/buildSrc/src/main/kotlin/ProjectConfig.kt b/buildSrc/src/main/kotlin/ProjectConfig.kt index 14969bfa5..c5fbc9d7f 100644 --- a/buildSrc/src/main/kotlin/ProjectConfig.kt +++ b/buildSrc/src/main/kotlin/ProjectConfig.kt @@ -55,6 +55,6 @@ const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app" // vc=19 / 0.1.0-AE — rust pipeline cutover. Extraction via // strawcore-core (Sulkta-Coop/strawcore) via the UniFFI wrapper; no // NewPipeExtractor in the runtime path. -const val STRAW_VERSION_CODE = 23 -const val STRAW_VERSION_NAME = "0.1.0-AI" +const val STRAW_VERSION_CODE = 24 +const val STRAW_VERSION_NAME = "0.1.0-AJ" const val STRAW_APPLICATION_ID = "com.sulkta.straw" diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt new file mode 100644 index 000000000..4e138788e --- /dev/null +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/dataimport/SettingsImport.kt @@ -0,0 +1,419 @@ +/* + * SPDX-FileCopyrightText: 2026 Sulkta-Coop + * SPDX-License-Identifier: GPL-3.0-or-later + * + * NewPipe / Tubular export importer. + * + * The user picks an exported `.zip` (NewPipe writes it as + * `NewPipeData-.zip`, Tubular as `TubularData-.zip`). + * Inside: + * - newpipe.db Room SQLite (subscriptions, playlists, history…) + * - preferences.json flat key/value of all user settings + * - newpipe.settings superseded XML form of preferences (we ignore) + * + * We populate Straw's existing stores (Subscriptions, Playlists, History, + * Settings) — filtering to service_id=0 (YouTube). Other services + * (SoundCloud / PeerTube / …) are silently dropped — we don't support + * them and a mixed import would surprise the user later. + * + * Resume positions (NewPipe `stream_state` table) are read but + * intentionally not persisted yet — Straw has no resume-positions + * store. Counted in [ImportResult.resumePositionsSeen] so the user + * knows the data was present even if dropped. + */ + +package com.sulkta.straw.feature.dataimport + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import android.net.Uri +import com.sulkta.straw.data.ChannelRef +import com.sulkta.straw.data.History +import com.sulkta.straw.data.PlaylistItem +import com.sulkta.straw.data.Playlists +import com.sulkta.straw.data.SbCategory +import com.sulkta.straw.data.Settings +import com.sulkta.straw.data.Subscriptions +import com.sulkta.straw.data.WatchHistoryItem +import java.io.File +import java.util.zip.ZipInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonPrimitive + +data class ImportResult( + val subscriptionsAdded: Int, + val subscriptionsSkippedNonYt: Int, + val playlistsAdded: Int, + val playlistItemsAdded: Int, + val searchHistoryAdded: Int, + val searchHistoryAvailable: Int, + val watchHistoryAdded: Int, + val watchHistoryAvailable: Int, + val resumePositionsSeen: Int, + val settingsApplied: Int, + val warnings: List, +) { + fun summary(): String = buildString { + append("Imported ") + append(subscriptionsAdded) + append(" subs") + if (subscriptionsSkippedNonYt > 0) { + append(" (skipped ") + append(subscriptionsSkippedNonYt) + append(" non-YouTube)") + } + append(", ") + append(playlistsAdded) + append(" playlist") + if (playlistsAdded != 1) append("s") + append(" (") + append(playlistItemsAdded) + append(" videos), ") + append(watchHistoryAdded) + append("/") + append(watchHistoryAvailable) + append(" watch history, ") + append(searchHistoryAdded) + append("/") + append(searchHistoryAvailable) + append(" searches, ") + append(settingsApplied) + append(" settings.") + if (resumePositionsSeen > 0) { + append(" Resume positions (") + append(resumePositionsSeen) + append(") not yet supported — dropped.") + } + if (warnings.isNotEmpty()) { + append("\n\nWarnings:\n") + warnings.forEach { append("• "); append(it); append("\n") } + } + } +} + +object SettingsImport { + + // YouTube only — Straw doesn't extract from other services. + private const val YT_SERVICE_ID = 0 + + suspend fun run(context: Context, zipUri: Uri): Result = + withContext(Dispatchers.IO) { + runCatching { runInner(context, zipUri) } + } + + private fun runInner(context: Context, zipUri: Uri): ImportResult { + val warnings = mutableListOf() + val workDir = File(context.cacheDir, "newpipe-import-${System.currentTimeMillis()}") + workDir.mkdirs() + try { + val (dbFile, prefsJson) = extractZip(context, zipUri, workDir, warnings) + + val subsResult = if (dbFile != null) importSubscriptions(dbFile) else SubsResult(0, 0) + val plResult = if (dbFile != null) importPlaylists(dbFile) else PlResult(0, 0) + val histResult = if (dbFile != null) importHistory(dbFile) else HistResult(0, 0, 0, 0, 0) + val settingsResult = if (prefsJson != null) importSettings(prefsJson) else 0 + + return ImportResult( + subscriptionsAdded = subsResult.added, + subscriptionsSkippedNonYt = subsResult.skipped, + playlistsAdded = plResult.playlists, + playlistItemsAdded = plResult.items, + searchHistoryAdded = histResult.searches, + searchHistoryAvailable = histResult.searchesAvailable, + watchHistoryAdded = histResult.watchesAdded, + watchHistoryAvailable = histResult.watchesAvailable, + resumePositionsSeen = histResult.resumePositions, + settingsApplied = settingsResult, + warnings = warnings, + ) + } finally { + workDir.deleteRecursively() + } + } + + private fun extractZip( + context: Context, + zipUri: Uri, + workDir: File, + warnings: MutableList, + ): Pair { + var dbFile: File? = null + var prefs: JsonObject? = null + context.contentResolver.openInputStream(zipUri)?.use { input -> + ZipInputStream(input).use { zip -> + while (true) { + val entry = zip.nextEntry ?: break + when (entry.name) { + "newpipe.db" -> { + val out = File(workDir, "newpipe.db") + out.outputStream().use { os -> + zip.copyTo(os, bufferSize = 64 * 1024) + } + dbFile = out + } + "preferences.json" -> { + val bytes = zip.readBytes() + prefs = runCatching { + Json.parseToJsonElement(bytes.decodeToString()) as? JsonObject + }.getOrNull() + if (prefs == null) warnings += "preferences.json present but unparseable" + } + // newpipe.settings is the legacy XML form; preferences.json + // supersedes it in every modern export. Skip. + else -> { /* ignore other entries */ } + } + zip.closeEntry() + } + } + } ?: error("Could not open the selected file") + if (dbFile == null) warnings += "newpipe.db not found in archive — most data skipped" + if (prefs == null) warnings += "preferences.json not found — settings not migrated" + return dbFile to prefs + } + + private data class SubsResult(val added: Int, val skipped: Int) + private fun importSubscriptions(dbFile: File): SubsResult { + val store = Subscriptions.get() + var added = 0 + var skipped = 0 + openDb(dbFile).use { db -> + db.rawQuery( + "SELECT url, name, avatar_url, service_id FROM subscriptions", + null, + ).use { c -> + while (c.moveToNext()) { + val serviceId = c.getInt(3) + if (serviceId != YT_SERVICE_ID) { + skipped++ + continue + } + val url = c.getString(0) ?: continue + val name = c.getString(1) ?: continue + val avatar = c.getString(2) + if (!store.isSubscribed(url)) { + store.toggle(ChannelRef(url = url, name = name, avatar = avatar)) + added++ + } + } + } + } + return SubsResult(added, skipped) + } + + private data class PlResult(val playlists: Int, val items: Int) + private fun importPlaylists(dbFile: File): PlResult { + val store = Playlists.get() + var playlistsAdded = 0 + var itemsAdded = 0 + openDb(dbFile).use { db -> + val playlistRows = mutableListOf>() + db.rawQuery("SELECT uid, name FROM playlists", null).use { c -> + while (c.moveToNext()) { + val uid = c.getLong(0) + val name = c.getString(1) ?: "Untitled" + playlistRows += uid to name + } + } + for ((uid, name) in playlistRows) { + val items = mutableListOf() + db.rawQuery( + """ + SELECT s.url, s.title, s.thumbnail_url, s.uploader, s.service_id + FROM playlist_stream_join j + JOIN streams s ON s.uid = j.stream_id + WHERE j.playlist_id = ? + ORDER BY j.join_index + """.trimIndent(), + arrayOf(uid.toString()), + ).use { c -> + while (c.moveToNext()) { + if (c.getInt(4) != YT_SERVICE_ID) continue + items += PlaylistItem( + streamUrl = c.getString(0) ?: continue, + title = c.getString(1) ?: "(no title)", + thumbnail = c.getString(2), + uploader = c.getString(3) ?: "", + addedAt = System.currentTimeMillis(), + ) + } + } + if (items.isEmpty()) continue + // Use the store's normal create + addItem rather than minting + // a Playlist directly — keeps the atomic-update path + // consistent with user-driven creates. + val created = store.create(name) + for (it in items) store.addItem(created.id, it) + playlistsAdded++ + itemsAdded += items.size + } + } + return PlResult(playlistsAdded, itemsAdded) + } + + private data class HistResult( + val watchesAdded: Int, + val watchesAvailable: Int, + val searches: Int, + val searchesAvailable: Int, + val resumePositions: Int, + ) + + private fun importHistory(dbFile: File): HistResult { + val historyStore = History.get() + var watchesSeen = 0 + var watchesAvailable = 0 + var searchesSeen = 0 + var resumePositions = 0 + val searchesBefore = historyStore.searches.value.size + val watchesBefore = historyStore.watches.value.size + openDb(dbFile).use { db -> + // Search history — feed oldest first so the store ends up with + // the most-recent on top after its own dedup + take(MAX). + db.rawQuery( + "SELECT search FROM search_history WHERE service_id=? ORDER BY creation_date ASC", + arrayOf(YT_SERVICE_ID.toString()), + ).use { c -> + while (c.moveToNext()) { + val q = c.getString(0) ?: continue + historyStore.recordSearch(q) + searchesSeen++ + } + } + + // Watch history — newest first via stream_history.access_date, + // joined to streams for the metadata we need. + // recordWatch caps internally; we just stop counting "added" once + // we've replayed Straw's MAX rows. (The store reverses to put + // most-recent on top — so we feed it oldest-first to match.) + db.rawQuery("SELECT COUNT(*) FROM stream_history", null).use { c -> + if (c.moveToNext()) watchesAvailable = c.getInt(0) + } + db.rawQuery( + """ + SELECT s.url, s.title, s.uploader, s.thumbnail_url, h.access_date, s.service_id + FROM stream_history h + JOIN streams s ON s.uid = h.stream_id + ORDER BY h.access_date ASC + """.trimIndent(), + null, + ).use { c -> + while (c.moveToNext()) { + if (c.getInt(5) != YT_SERVICE_ID) continue + val url = c.getString(0) ?: continue + val title = c.getString(1) ?: continue + val uploader = c.getString(2) ?: "" + val thumb = c.getString(3) + val videoId = extractYtVideoId(url) ?: continue + historyStore.recordWatch( + WatchHistoryItem( + url = url, + videoId = videoId, + title = title, + uploader = uploader, + thumbnail = thumb, + watchedAt = c.getLong(4), + ), + ) + watchesSeen++ + } + } + + // Resume positions — counted, not stored. Future task hooks into + // a ResumePositionsStore. + db.rawQuery("SELECT COUNT(*) FROM stream_state", null).use { c -> + if (c.moveToNext()) resumePositions = c.getInt(0) + } + } + // Report what actually landed in the store after its dedup + caps. + return HistResult( + watchesAdded = historyStore.watches.value.size - watchesBefore, + watchesAvailable = watchesAvailable.takeIf { it > 0 } ?: watchesSeen, + searches = historyStore.searches.value.size - searchesBefore, + resumePositions = resumePositions, + searchesAvailable = searchesSeen, + ) + } + + private fun importSettings(prefs: JsonObject): Int { + val settings = Settings.get() + var applied = 0 + + // SponsorBlock: master toggle gates the categories. If disabled in + // NewPipe, leave Straw's categories alone (they have a non-empty + // default). If enabled, sync each category boolean. + val sbMaster = prefs.boolOrNull("sponsor_block_enable") + if (sbMaster == true) { + val targets = mapOf( + "sponsor_block_category_sponsor" to SbCategory.Sponsor, + "sponsor_block_category_self_promo" to SbCategory.SelfPromo, + "sponsor_block_category_intro" to SbCategory.Intro, + "sponsor_block_category_outro" to SbCategory.Outro, + "sponsor_block_category_interaction" to SbCategory.Interaction, + "sponsor_block_category_music" to SbCategory.MusicOfftopic, + "sponsor_block_category_filler" to SbCategory.Filler, + ) + val current = settings.sbCategories.value + for ((key, cat) in targets) { + val want = prefs.boolOrNull(key) ?: continue + val have = cat in current + if (want != have) settings.toggle(cat) + applied++ + } + } + + // Default resolution: NewPipe values like "720p60", "1080p", "Best + // resolution". Map down to Straw's discrete ceilings. + prefs.stringOrNull("default_resolution")?.let { raw -> + val r = parseResolution(raw) + if (r != null) { + settings.setMaxResolution(r) + applied++ + } + } + + return applied + } + + private fun parseResolution(raw: String): com.sulkta.straw.data.MaxResolution? { + val n = Regex("(\\d+)").find(raw)?.groupValues?.get(1)?.toIntOrNull() + ?: return when (raw.lowercase()) { + "best resolution", "best", "highest" -> com.sulkta.straw.data.MaxResolution.Auto + else -> null + } + return when { + n >= 1080 -> com.sulkta.straw.data.MaxResolution.P1080 + n >= 720 -> com.sulkta.straw.data.MaxResolution.P720 + n >= 480 -> com.sulkta.straw.data.MaxResolution.P480 + n >= 360 -> com.sulkta.straw.data.MaxResolution.P360 + else -> com.sulkta.straw.data.MaxResolution.P144 + } + } + + private fun openDb(dbFile: File): SQLiteDatabase = + SQLiteDatabase.openDatabase( + dbFile.absolutePath, + /* factory = */ null, + SQLiteDatabase.OPEN_READONLY, + ) + + // YouTube URL patterns we need to parse for the videoId column on + // WatchHistoryItem. Cover the watch?v= form (canonical), youtu.be + // shortlinks, and embed/. Reject anything we can't parse rather than + // inventing IDs. + private val YT_ID = Regex( + "(?:youtu\\.be/|youtube(?:-nocookie)?\\.com/(?:watch\\?(?:.*&)?v=|embed/|v/|shorts/))([A-Za-z0-9_-]{6,15})", + ) + private fun extractYtVideoId(url: String): String? = + YT_ID.find(url)?.groupValues?.get(1) + + private fun JsonObject.boolOrNull(key: String): Boolean? = + runCatching { this[key]?.jsonPrimitive?.boolean }.getOrNull() + + private fun JsonObject.stringOrNull(key: String): String? = + runCatching { this[key]?.jsonPrimitive?.contentOrNull }.getOrNull() +} diff --git a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt index f090fc99c..f3e704b0d 100644 --- a/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt +++ b/strawApp/src/main/kotlin/com/sulkta/straw/feature/settings/SettingsScreen.kt @@ -5,41 +5,66 @@ package com.sulkta.straw.feature.settings +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.statusBarsPadding import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.statusBarsPadding import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Switch import androidx.compose.material3.Text -import androidx.compose.foundation.clickable -import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.runtime.collectAsState +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import com.sulkta.straw.data.History import com.sulkta.straw.data.MaxResolution import com.sulkta.straw.data.SbCategory import com.sulkta.straw.data.Settings +import com.sulkta.straw.feature.dataimport.ImportResult +import com.sulkta.straw.feature.dataimport.SettingsImport +import kotlinx.coroutines.launch @Composable fun SettingsScreen() { val store = Settings.get() val cats by store.sbCategories.collectAsState() + val context = LocalContext.current + val scope = rememberCoroutineScope() + + var importRunning by remember { mutableStateOf(false) } + var importResult by remember { mutableStateOf?>(null) } + val pickZip = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri -> + if (uri == null) return@rememberLauncherForActivityResult + importRunning = true + scope.launch { + importResult = SettingsImport.run(context, uri) + importRunning = false + } + } Column( modifier = Modifier @@ -124,6 +149,54 @@ fun SettingsScreen() { Text("Clear searches") } } + + Spacer(modifier = Modifier.height(32.dp)) + Text( + "Import from NewPipe / Tubular", + style = MaterialTheme.typography.titleMedium, + fontWeight = FontWeight.SemiBold, + ) + Spacer(modifier = Modifier.height(4.dp)) + Text( + "Pick a NewPipeData-*.zip or TubularData-*.zip — we'll lift " + + "your subscriptions, playlists, search history, watch history " + + "(capped to 50 most recent), and a curated subset of settings. " + + "Other services (SoundCloud, PeerTube) are skipped.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Spacer(modifier = Modifier.height(12.dp)) + Button( + enabled = !importRunning, + onClick = { pickZip.launch(arrayOf("application/zip", "application/octet-stream", "*/*")) }, + ) { + if (importRunning) { + CircularProgressIndicator( + modifier = Modifier.height(18.dp).padding(end = 8.dp), + strokeWidth = 2.dp, + ) + Text("Importing…") + } else { + Text("Pick export file…") + } + } + } + + importResult?.let { res -> + AlertDialog( + onDismissRequest = { importResult = null }, + title = { Text(if (res.isSuccess) "Import complete" else "Import failed") }, + text = { + val body = res.fold( + onSuccess = { it.summary() }, + onFailure = { it.message ?: it.javaClass.simpleName }, + ) + Text(body, style = MaterialTheme.typography.bodyMedium) + }, + confirmButton = { + TextButton(onClick = { importResult = null }) { Text("OK") } + }, + ) } }