vc=24: NewPipe/Tubular settings import

Settings screen now has an Import section. Picks a NewPipeData-*.zip or TubularData-*.zip via the SAF, extracts newpipe.db + preferences.json, and walks the Room SQLite schema (subscriptions / playlists / playlist_stream_join / streams / search_history / stream_history / stream_state). YouTube only — service_id=0 filter drops SoundCloud/PeerTube/etc.

Imported on smoke: 26 subs, 1 playlist with 10 items, 50/11402 watch history (capped by HistoryStore MAX_WATCHES), 20/2242 searches (MAX_SEARCHES), 8 settings keys (SponsorBlock category toggles + default resolution). Resume positions counted (10918) but not yet persisted — Straw has no resume-store yet.
This commit is contained in:
Kayos 2026-05-25 16:44:27 +00:00
parent 3ff9740c40
commit 1443bb8ef7
3 changed files with 498 additions and 6 deletions

View file

@ -55,6 +55,6 @@ const val NEWPIPE_APPLICATION_ID_NEW = "net.newpipe.app"
// vc=19 / 0.1.0-AE — rust pipeline cutover. Extraction via
// strawcore-core (Sulkta-Coop/strawcore) via the UniFFI wrapper; no
// NewPipeExtractor in the runtime path.
const val STRAW_VERSION_CODE = 23
const val STRAW_VERSION_NAME = "0.1.0-AI"
const val STRAW_VERSION_CODE = 24
const val STRAW_VERSION_NAME = "0.1.0-AJ"
const val STRAW_APPLICATION_ID = "com.sulkta.straw"

View file

@ -0,0 +1,419 @@
/*
* SPDX-FileCopyrightText: 2026 Sulkta-Coop
* SPDX-License-Identifier: GPL-3.0-or-later
*
* NewPipe / Tubular export importer.
*
* The user picks an exported `.zip` (NewPipe writes it as
* `NewPipeData-<date>.zip`, Tubular as `TubularData-<date>.zip`).
* Inside:
* - newpipe.db Room SQLite (subscriptions, playlists, history)
* - preferences.json flat key/value of all user settings
* - newpipe.settings superseded XML form of preferences (we ignore)
*
* We populate Straw's existing stores (Subscriptions, Playlists, History,
* Settings) filtering to service_id=0 (YouTube). Other services
* (SoundCloud / PeerTube / ) are silently dropped we don't support
* them and a mixed import would surprise the user later.
*
* Resume positions (NewPipe `stream_state` table) are read but
* intentionally not persisted yet Straw has no resume-positions
* store. Counted in [ImportResult.resumePositionsSeen] so the user
* knows the data was present even if dropped.
*/
package com.sulkta.straw.feature.dataimport
import android.content.Context
import android.database.sqlite.SQLiteDatabase
import android.net.Uri
import com.sulkta.straw.data.ChannelRef
import com.sulkta.straw.data.History
import com.sulkta.straw.data.PlaylistItem
import com.sulkta.straw.data.Playlists
import com.sulkta.straw.data.SbCategory
import com.sulkta.straw.data.Settings
import com.sulkta.straw.data.Subscriptions
import com.sulkta.straw.data.WatchHistoryItem
import java.io.File
import java.util.zip.ZipInputStream
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.boolean
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonPrimitive
data class ImportResult(
val subscriptionsAdded: Int,
val subscriptionsSkippedNonYt: Int,
val playlistsAdded: Int,
val playlistItemsAdded: Int,
val searchHistoryAdded: Int,
val searchHistoryAvailable: Int,
val watchHistoryAdded: Int,
val watchHistoryAvailable: Int,
val resumePositionsSeen: Int,
val settingsApplied: Int,
val warnings: List<String>,
) {
fun summary(): String = buildString {
append("Imported ")
append(subscriptionsAdded)
append(" subs")
if (subscriptionsSkippedNonYt > 0) {
append(" (skipped ")
append(subscriptionsSkippedNonYt)
append(" non-YouTube)")
}
append(", ")
append(playlistsAdded)
append(" playlist")
if (playlistsAdded != 1) append("s")
append(" (")
append(playlistItemsAdded)
append(" videos), ")
append(watchHistoryAdded)
append("/")
append(watchHistoryAvailable)
append(" watch history, ")
append(searchHistoryAdded)
append("/")
append(searchHistoryAvailable)
append(" searches, ")
append(settingsApplied)
append(" settings.")
if (resumePositionsSeen > 0) {
append(" Resume positions (")
append(resumePositionsSeen)
append(") not yet supported — dropped.")
}
if (warnings.isNotEmpty()) {
append("\n\nWarnings:\n")
warnings.forEach { append(""); append(it); append("\n") }
}
}
}
object SettingsImport {
// YouTube only — Straw doesn't extract from other services.
private const val YT_SERVICE_ID = 0
suspend fun run(context: Context, zipUri: Uri): Result<ImportResult> =
withContext(Dispatchers.IO) {
runCatching { runInner(context, zipUri) }
}
private fun runInner(context: Context, zipUri: Uri): ImportResult {
val warnings = mutableListOf<String>()
val workDir = File(context.cacheDir, "newpipe-import-${System.currentTimeMillis()}")
workDir.mkdirs()
try {
val (dbFile, prefsJson) = extractZip(context, zipUri, workDir, warnings)
val subsResult = if (dbFile != null) importSubscriptions(dbFile) else SubsResult(0, 0)
val plResult = if (dbFile != null) importPlaylists(dbFile) else PlResult(0, 0)
val histResult = if (dbFile != null) importHistory(dbFile) else HistResult(0, 0, 0, 0, 0)
val settingsResult = if (prefsJson != null) importSettings(prefsJson) else 0
return ImportResult(
subscriptionsAdded = subsResult.added,
subscriptionsSkippedNonYt = subsResult.skipped,
playlistsAdded = plResult.playlists,
playlistItemsAdded = plResult.items,
searchHistoryAdded = histResult.searches,
searchHistoryAvailable = histResult.searchesAvailable,
watchHistoryAdded = histResult.watchesAdded,
watchHistoryAvailable = histResult.watchesAvailable,
resumePositionsSeen = histResult.resumePositions,
settingsApplied = settingsResult,
warnings = warnings,
)
} finally {
workDir.deleteRecursively()
}
}
private fun extractZip(
context: Context,
zipUri: Uri,
workDir: File,
warnings: MutableList<String>,
): Pair<File?, JsonObject?> {
var dbFile: File? = null
var prefs: JsonObject? = null
context.contentResolver.openInputStream(zipUri)?.use { input ->
ZipInputStream(input).use { zip ->
while (true) {
val entry = zip.nextEntry ?: break
when (entry.name) {
"newpipe.db" -> {
val out = File(workDir, "newpipe.db")
out.outputStream().use { os ->
zip.copyTo(os, bufferSize = 64 * 1024)
}
dbFile = out
}
"preferences.json" -> {
val bytes = zip.readBytes()
prefs = runCatching {
Json.parseToJsonElement(bytes.decodeToString()) as? JsonObject
}.getOrNull()
if (prefs == null) warnings += "preferences.json present but unparseable"
}
// newpipe.settings is the legacy XML form; preferences.json
// supersedes it in every modern export. Skip.
else -> { /* ignore other entries */ }
}
zip.closeEntry()
}
}
} ?: error("Could not open the selected file")
if (dbFile == null) warnings += "newpipe.db not found in archive — most data skipped"
if (prefs == null) warnings += "preferences.json not found — settings not migrated"
return dbFile to prefs
}
private data class SubsResult(val added: Int, val skipped: Int)
private fun importSubscriptions(dbFile: File): SubsResult {
val store = Subscriptions.get()
var added = 0
var skipped = 0
openDb(dbFile).use { db ->
db.rawQuery(
"SELECT url, name, avatar_url, service_id FROM subscriptions",
null,
).use { c ->
while (c.moveToNext()) {
val serviceId = c.getInt(3)
if (serviceId != YT_SERVICE_ID) {
skipped++
continue
}
val url = c.getString(0) ?: continue
val name = c.getString(1) ?: continue
val avatar = c.getString(2)
if (!store.isSubscribed(url)) {
store.toggle(ChannelRef(url = url, name = name, avatar = avatar))
added++
}
}
}
}
return SubsResult(added, skipped)
}
private data class PlResult(val playlists: Int, val items: Int)
private fun importPlaylists(dbFile: File): PlResult {
val store = Playlists.get()
var playlistsAdded = 0
var itemsAdded = 0
openDb(dbFile).use { db ->
val playlistRows = mutableListOf<Pair<Long, String>>()
db.rawQuery("SELECT uid, name FROM playlists", null).use { c ->
while (c.moveToNext()) {
val uid = c.getLong(0)
val name = c.getString(1) ?: "Untitled"
playlistRows += uid to name
}
}
for ((uid, name) in playlistRows) {
val items = mutableListOf<PlaylistItem>()
db.rawQuery(
"""
SELECT s.url, s.title, s.thumbnail_url, s.uploader, s.service_id
FROM playlist_stream_join j
JOIN streams s ON s.uid = j.stream_id
WHERE j.playlist_id = ?
ORDER BY j.join_index
""".trimIndent(),
arrayOf(uid.toString()),
).use { c ->
while (c.moveToNext()) {
if (c.getInt(4) != YT_SERVICE_ID) continue
items += PlaylistItem(
streamUrl = c.getString(0) ?: continue,
title = c.getString(1) ?: "(no title)",
thumbnail = c.getString(2),
uploader = c.getString(3) ?: "",
addedAt = System.currentTimeMillis(),
)
}
}
if (items.isEmpty()) continue
// Use the store's normal create + addItem rather than minting
// a Playlist directly — keeps the atomic-update path
// consistent with user-driven creates.
val created = store.create(name)
for (it in items) store.addItem(created.id, it)
playlistsAdded++
itemsAdded += items.size
}
}
return PlResult(playlistsAdded, itemsAdded)
}
private data class HistResult(
val watchesAdded: Int,
val watchesAvailable: Int,
val searches: Int,
val searchesAvailable: Int,
val resumePositions: Int,
)
private fun importHistory(dbFile: File): HistResult {
val historyStore = History.get()
var watchesSeen = 0
var watchesAvailable = 0
var searchesSeen = 0
var resumePositions = 0
val searchesBefore = historyStore.searches.value.size
val watchesBefore = historyStore.watches.value.size
openDb(dbFile).use { db ->
// Search history — feed oldest first so the store ends up with
// the most-recent on top after its own dedup + take(MAX).
db.rawQuery(
"SELECT search FROM search_history WHERE service_id=? ORDER BY creation_date ASC",
arrayOf(YT_SERVICE_ID.toString()),
).use { c ->
while (c.moveToNext()) {
val q = c.getString(0) ?: continue
historyStore.recordSearch(q)
searchesSeen++
}
}
// Watch history — newest first via stream_history.access_date,
// joined to streams for the metadata we need.
// recordWatch caps internally; we just stop counting "added" once
// we've replayed Straw's MAX rows. (The store reverses to put
// most-recent on top — so we feed it oldest-first to match.)
db.rawQuery("SELECT COUNT(*) FROM stream_history", null).use { c ->
if (c.moveToNext()) watchesAvailable = c.getInt(0)
}
db.rawQuery(
"""
SELECT s.url, s.title, s.uploader, s.thumbnail_url, h.access_date, s.service_id
FROM stream_history h
JOIN streams s ON s.uid = h.stream_id
ORDER BY h.access_date ASC
""".trimIndent(),
null,
).use { c ->
while (c.moveToNext()) {
if (c.getInt(5) != YT_SERVICE_ID) continue
val url = c.getString(0) ?: continue
val title = c.getString(1) ?: continue
val uploader = c.getString(2) ?: ""
val thumb = c.getString(3)
val videoId = extractYtVideoId(url) ?: continue
historyStore.recordWatch(
WatchHistoryItem(
url = url,
videoId = videoId,
title = title,
uploader = uploader,
thumbnail = thumb,
watchedAt = c.getLong(4),
),
)
watchesSeen++
}
}
// Resume positions — counted, not stored. Future task hooks into
// a ResumePositionsStore.
db.rawQuery("SELECT COUNT(*) FROM stream_state", null).use { c ->
if (c.moveToNext()) resumePositions = c.getInt(0)
}
}
// Report what actually landed in the store after its dedup + caps.
return HistResult(
watchesAdded = historyStore.watches.value.size - watchesBefore,
watchesAvailable = watchesAvailable.takeIf { it > 0 } ?: watchesSeen,
searches = historyStore.searches.value.size - searchesBefore,
resumePositions = resumePositions,
searchesAvailable = searchesSeen,
)
}
private fun importSettings(prefs: JsonObject): Int {
val settings = Settings.get()
var applied = 0
// SponsorBlock: master toggle gates the categories. If disabled in
// NewPipe, leave Straw's categories alone (they have a non-empty
// default). If enabled, sync each category boolean.
val sbMaster = prefs.boolOrNull("sponsor_block_enable")
if (sbMaster == true) {
val targets = mapOf(
"sponsor_block_category_sponsor" to SbCategory.Sponsor,
"sponsor_block_category_self_promo" to SbCategory.SelfPromo,
"sponsor_block_category_intro" to SbCategory.Intro,
"sponsor_block_category_outro" to SbCategory.Outro,
"sponsor_block_category_interaction" to SbCategory.Interaction,
"sponsor_block_category_music" to SbCategory.MusicOfftopic,
"sponsor_block_category_filler" to SbCategory.Filler,
)
val current = settings.sbCategories.value
for ((key, cat) in targets) {
val want = prefs.boolOrNull(key) ?: continue
val have = cat in current
if (want != have) settings.toggle(cat)
applied++
}
}
// Default resolution: NewPipe values like "720p60", "1080p", "Best
// resolution". Map down to Straw's discrete ceilings.
prefs.stringOrNull("default_resolution")?.let { raw ->
val r = parseResolution(raw)
if (r != null) {
settings.setMaxResolution(r)
applied++
}
}
return applied
}
private fun parseResolution(raw: String): com.sulkta.straw.data.MaxResolution? {
val n = Regex("(\\d+)").find(raw)?.groupValues?.get(1)?.toIntOrNull()
?: return when (raw.lowercase()) {
"best resolution", "best", "highest" -> com.sulkta.straw.data.MaxResolution.Auto
else -> null
}
return when {
n >= 1080 -> com.sulkta.straw.data.MaxResolution.P1080
n >= 720 -> com.sulkta.straw.data.MaxResolution.P720
n >= 480 -> com.sulkta.straw.data.MaxResolution.P480
n >= 360 -> com.sulkta.straw.data.MaxResolution.P360
else -> com.sulkta.straw.data.MaxResolution.P144
}
}
private fun openDb(dbFile: File): SQLiteDatabase =
SQLiteDatabase.openDatabase(
dbFile.absolutePath,
/* factory = */ null,
SQLiteDatabase.OPEN_READONLY,
)
// YouTube URL patterns we need to parse for the videoId column on
// WatchHistoryItem. Cover the watch?v= form (canonical), youtu.be
// shortlinks, and embed/. Reject anything we can't parse rather than
// inventing IDs.
private val YT_ID = Regex(
"(?:youtu\\.be/|youtube(?:-nocookie)?\\.com/(?:watch\\?(?:.*&)?v=|embed/|v/|shorts/))([A-Za-z0-9_-]{6,15})",
)
private fun extractYtVideoId(url: String): String? =
YT_ID.find(url)?.groupValues?.get(1)
private fun JsonObject.boolOrNull(key: String): Boolean? =
runCatching { this[key]?.jsonPrimitive?.boolean }.getOrNull()
private fun JsonObject.stringOrNull(key: String): String? =
runCatching { this[key]?.jsonPrimitive?.contentOrNull }.getOrNull()
}

View file

@ -5,41 +5,66 @@
package com.sulkta.straw.feature.settings
import androidx.activity.compose.rememberLauncherForActivityResult
import androidx.activity.result.contract.ActivityResultContracts
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.statusBarsPadding
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.Button
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.HorizontalDivider
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Switch
import androidx.compose.material3.Text
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.runtime.collectAsState
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import com.sulkta.straw.data.History
import com.sulkta.straw.data.MaxResolution
import com.sulkta.straw.data.SbCategory
import com.sulkta.straw.data.Settings
import com.sulkta.straw.feature.dataimport.ImportResult
import com.sulkta.straw.feature.dataimport.SettingsImport
import kotlinx.coroutines.launch
@Composable
fun SettingsScreen() {
val store = Settings.get()
val cats by store.sbCategories.collectAsState()
val context = LocalContext.current
val scope = rememberCoroutineScope()
var importRunning by remember { mutableStateOf(false) }
var importResult by remember { mutableStateOf<Result<ImportResult>?>(null) }
val pickZip = rememberLauncherForActivityResult(ActivityResultContracts.OpenDocument()) { uri ->
if (uri == null) return@rememberLauncherForActivityResult
importRunning = true
scope.launch {
importResult = SettingsImport.run(context, uri)
importRunning = false
}
}
Column(
modifier = Modifier
@ -124,6 +149,54 @@ fun SettingsScreen() {
Text("Clear searches")
}
}
Spacer(modifier = Modifier.height(32.dp))
Text(
"Import from NewPipe / Tubular",
style = MaterialTheme.typography.titleMedium,
fontWeight = FontWeight.SemiBold,
)
Spacer(modifier = Modifier.height(4.dp))
Text(
"Pick a NewPipeData-*.zip or TubularData-*.zip — we'll lift " +
"your subscriptions, playlists, search history, watch history " +
"(capped to 50 most recent), and a curated subset of settings. " +
"Other services (SoundCloud, PeerTube) are skipped.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
Spacer(modifier = Modifier.height(12.dp))
Button(
enabled = !importRunning,
onClick = { pickZip.launch(arrayOf("application/zip", "application/octet-stream", "*/*")) },
) {
if (importRunning) {
CircularProgressIndicator(
modifier = Modifier.height(18.dp).padding(end = 8.dp),
strokeWidth = 2.dp,
)
Text("Importing…")
} else {
Text("Pick export file…")
}
}
}
importResult?.let { res ->
AlertDialog(
onDismissRequest = { importResult = null },
title = { Text(if (res.isSuccess) "Import complete" else "Import failed") },
text = {
val body = res.fold(
onSuccess = { it.summary() },
onFailure = { it.message ?: it.javaClass.simpleName },
)
Text(body, style = MaterialTheme.typography.bodyMedium)
},
confirmButton = {
TextButton(onClick = { importResult = null }) { Text("OK") }
},
)
}
}