Load only the selected group and customizable updated status timeout
Now only the subscriptions from the selected group by the user will be loaded. Also add an option to decide how much time have to pass since the last refresh before the subscription is deemed as not up to date. This helps when a subscription appear in multiple groups, since updating in one will not require to be fetched again in the others.
This commit is contained in:
Mauricio Colli
parent
2948e4190b
commit
b2f317ab7c
20 changed files with 412 additions and 123 deletions
|
|
@ -9,6 +9,7 @@ import org.schabi.newpipe.database.feed.dao.FeedGroupDAO;
|
|||
import org.schabi.newpipe.database.feed.model.FeedEntity;
|
||||
import org.schabi.newpipe.database.feed.model.FeedGroupEntity;
|
||||
import org.schabi.newpipe.database.feed.model.FeedGroupSubscriptionEntity;
|
||||
import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity;
|
||||
import org.schabi.newpipe.database.history.dao.SearchHistoryDAO;
|
||||
import org.schabi.newpipe.database.history.dao.StreamHistoryDAO;
|
||||
import org.schabi.newpipe.database.history.model.SearchHistoryEntry;
|
||||
|
|
@ -34,7 +35,8 @@ import static org.schabi.newpipe.database.Migrations.DB_VER_3;
|
|||
SubscriptionEntity.class, SearchHistoryEntry.class,
|
||||
StreamEntity.class, StreamHistoryEntity.class, StreamStateEntity.class,
|
||||
PlaylistEntity.class, PlaylistStreamEntity.class, PlaylistRemoteEntity.class,
|
||||
FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class
|
||||
FeedEntity.class, FeedGroupEntity.class, FeedGroupSubscriptionEntity.class,
|
||||
FeedLastUpdatedEntity.class
|
||||
},
|
||||
version = DB_VER_3
|
||||
)
|
||||
|
|
|
|||
|
|
@ -78,10 +78,11 @@ public class Migrations {
|
|||
// Add NOT NULLs and new fields
|
||||
database.execSQL("CREATE TABLE IF NOT EXISTS streams_new " +
|
||||
"(uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, service_id INTEGER NOT NULL, url TEXT NOT NULL, title TEXT NOT NULL, stream_type TEXT NOT NULL," +
|
||||
" duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER)");
|
||||
" duration INTEGER NOT NULL, uploader TEXT NOT NULL, thumbnail_url TEXT, view_count INTEGER, textual_upload_date TEXT, upload_date INTEGER," +
|
||||
" is_upload_date_approximation INTEGER)");
|
||||
|
||||
database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date)"+
|
||||
" SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL FROM streams");
|
||||
database.execSQL("INSERT INTO streams_new (uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, view_count, textual_upload_date, upload_date, is_upload_date_approximation)"+
|
||||
" SELECT uid, service_id, url, title, stream_type, duration, uploader, thumbnail_url, NULL, NULL, NULL, NULL FROM streams");
|
||||
|
||||
database.execSQL("DROP TABLE streams");
|
||||
database.execSQL("ALTER TABLE streams_new RENAME TO streams");
|
||||
|
|
@ -93,6 +94,7 @@ public class Migrations {
|
|||
database.execSQL("CREATE TABLE IF NOT EXISTS feed_group (uid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, name TEXT NOT NULL, icon_id INTEGER NOT NULL)");
|
||||
database.execSQL("CREATE TABLE IF NOT EXISTS feed_group_subscription_join (group_id INTEGER NOT NULL, subscription_id INTEGER NOT NULL, PRIMARY KEY(group_id, subscription_id), FOREIGN KEY(group_id) REFERENCES feed_group(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)");
|
||||
database.execSQL("CREATE INDEX index_feed_group_subscription_join_subscription_id ON feed_group_subscription_join (subscription_id)");
|
||||
database.execSQL("CREATE TABLE IF NOT EXISTS feed_last_updated (subscription_id INTEGER NOT NULL, last_updated INTEGER, PRIMARY KEY(subscription_id), FOREIGN KEY(subscription_id) REFERENCES subscriptions(uid) ON UPDATE CASCADE ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED)");
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
package org.schabi.newpipe.database.feed.dao
|
||||
|
||||
import androidx.room.Dao
|
||||
import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import androidx.room.*
|
||||
import io.reactivex.Flowable
|
||||
import org.schabi.newpipe.database.feed.model.FeedEntity
|
||||
import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity
|
||||
import org.schabi.newpipe.database.stream.model.StreamEntity
|
||||
import org.schabi.newpipe.database.subscription.SubscriptionEntity
|
||||
import java.util.*
|
||||
|
||||
@Dao
|
||||
|
|
@ -80,4 +79,69 @@ abstract class FeedDAO {
|
|||
|
||||
@Insert(onConflict = OnConflictStrategy.IGNORE)
|
||||
abstract fun insertAll(entities: List<FeedEntity>): List<Long>
|
||||
|
||||
@Insert(onConflict = OnConflictStrategy.IGNORE)
|
||||
internal abstract fun insertLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity): Long
|
||||
|
||||
@Update(onConflict = OnConflictStrategy.IGNORE)
|
||||
internal abstract fun updateLastUpdated(lastUpdatedEntity: FeedLastUpdatedEntity)
|
||||
|
||||
@Transaction
|
||||
open fun setLastUpdatedForSubscription(lastUpdatedEntity: FeedLastUpdatedEntity) {
|
||||
val id = insertLastUpdated(lastUpdatedEntity)
|
||||
|
||||
if (id == -1L) {
|
||||
updateLastUpdated(lastUpdatedEntity)
|
||||
}
|
||||
}
|
||||
|
||||
@Query("""
|
||||
SELECT MIN(lu.last_updated) FROM feed_last_updated lu
|
||||
|
||||
INNER JOIN feed_group_subscription_join fgs
|
||||
ON fgs.subscription_id = lu.subscription_id AND fgs.group_id = :groupId
|
||||
""")
|
||||
abstract fun oldestSubscriptionUpdate(groupId: Long): Flowable<List<Date>>
|
||||
|
||||
@Query("SELECT MIN(last_updated) FROM feed_last_updated")
|
||||
abstract fun oldestSubscriptionUpdateFromAll(): Flowable<List<Date>>
|
||||
|
||||
@Query("SELECT COUNT(*) FROM feed_last_updated WHERE last_updated IS NULL")
|
||||
abstract fun notLoadedCount(): Flowable<Long>
|
||||
|
||||
@Query("""
|
||||
SELECT COUNT(*) FROM subscriptions s
|
||||
|
||||
INNER JOIN feed_group_subscription_join fgs
|
||||
ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId
|
||||
|
||||
LEFT JOIN feed_last_updated lu
|
||||
ON s.uid = lu.subscription_id
|
||||
|
||||
WHERE lu.last_updated IS NULL
|
||||
""")
|
||||
abstract fun notLoadedCountForGroup(groupId: Long): Flowable<Long>
|
||||
|
||||
@Query("""
|
||||
SELECT s.* FROM subscriptions s
|
||||
|
||||
LEFT JOIN feed_last_updated lu
|
||||
ON s.uid = lu.subscription_id
|
||||
|
||||
WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold
|
||||
""")
|
||||
abstract fun getAllOutdated(outdatedThreshold: Date): Flowable<List<SubscriptionEntity>>
|
||||
|
||||
@Query("""
|
||||
SELECT s.* FROM subscriptions s
|
||||
|
||||
INNER JOIN feed_group_subscription_join fgs
|
||||
ON s.uid = fgs.subscription_id AND fgs.group_id = :groupId
|
||||
|
||||
LEFT JOIN feed_last_updated lu
|
||||
ON s.uid = lu.subscription_id
|
||||
|
||||
WHERE lu.last_updated IS NULL OR lu.last_updated < :outdatedThreshold
|
||||
""")
|
||||
abstract fun getAllOutdatedForGroup(groupId: Long, outdatedThreshold: Date): Flowable<List<SubscriptionEntity>>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
package org.schabi.newpipe.database.feed.model
|
||||
|
||||
import androidx.room.ColumnInfo
|
||||
import androidx.room.Entity
|
||||
import androidx.room.ForeignKey
|
||||
import androidx.room.PrimaryKey
|
||||
import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.FEED_LAST_UPDATED_TABLE
|
||||
import org.schabi.newpipe.database.feed.model.FeedLastUpdatedEntity.Companion.SUBSCRIPTION_ID
|
||||
import org.schabi.newpipe.database.subscription.SubscriptionEntity
|
||||
import java.util.*
|
||||
|
||||
@Entity(
|
||||
tableName = FEED_LAST_UPDATED_TABLE,
|
||||
foreignKeys = [
|
||||
ForeignKey(
|
||||
entity = SubscriptionEntity::class,
|
||||
parentColumns = [SubscriptionEntity.SUBSCRIPTION_UID],
|
||||
childColumns = [SUBSCRIPTION_ID],
|
||||
onDelete = ForeignKey.CASCADE, onUpdate = ForeignKey.CASCADE, deferred = true)
|
||||
]
|
||||
)
|
||||
data class FeedLastUpdatedEntity(
|
||||
@PrimaryKey
|
||||
@ColumnInfo(name = SUBSCRIPTION_ID)
|
||||
var subscriptionId: Long,
|
||||
|
||||
@ColumnInfo(name = LAST_UPDATED)
|
||||
var lastUpdated: Date? = null
|
||||
) {
|
||||
|
||||
companion object {
|
||||
const val FEED_LAST_UPDATED_TABLE = "feed_last_updated"
|
||||
|
||||
const val SUBSCRIPTION_ID = "subscription_id"
|
||||
const val LAST_UPDATED = "last_updated"
|
||||
}
|
||||
}
|
||||
|
|
@ -6,7 +6,8 @@ import org.schabi.newpipe.database.BasicDAO
|
|||
import org.schabi.newpipe.database.stream.model.StreamEntity
|
||||
import org.schabi.newpipe.database.stream.model.StreamEntity.Companion.STREAM_ID
|
||||
import org.schabi.newpipe.extractor.stream.StreamType
|
||||
import org.schabi.newpipe.extractor.stream.StreamType.*
|
||||
import org.schabi.newpipe.extractor.stream.StreamType.AUDIO_LIVE_STREAM
|
||||
import org.schabi.newpipe.extractor.stream.StreamType.LIVE_STREAM
|
||||
import java.util.*
|
||||
import kotlin.collections.ArrayList
|
||||
|
||||
|
|
@ -31,8 +32,8 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
|
|||
internal abstract fun silentInsertAllInternal(streams: List<StreamEntity>): List<Long>
|
||||
|
||||
@Query("""
|
||||
SELECT uid, stream_type, textual_upload_date, upload_date FROM streams
|
||||
WHERE url = :url AND service_id = :serviceId
|
||||
SELECT uid, stream_type, textual_upload_date, upload_date, is_upload_date_approximation, duration
|
||||
FROM streams WHERE url = :url AND service_id = :serviceId
|
||||
""")
|
||||
internal abstract fun getMinimalStreamForCompare(serviceId: Int, url: String): StreamCompareFeed?
|
||||
|
||||
|
|
@ -79,8 +80,16 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
|
|||
|
||||
val isNewerStreamLive = newerStream.streamType == AUDIO_LIVE_STREAM || newerStream.streamType == LIVE_STREAM
|
||||
if (!isNewerStreamLive) {
|
||||
if (existentMinimalStream.uploadDate != null) newerStream.uploadDate = existentMinimalStream.uploadDate
|
||||
if (existentMinimalStream.textualUploadDate != null) newerStream.textualUploadDate = existentMinimalStream.textualUploadDate
|
||||
if (existentMinimalStream.uploadDate != null && existentMinimalStream.isUploadDateApproximation != true) {
|
||||
newerStream.uploadDate = existentMinimalStream.uploadDate
|
||||
newerStream.textualUploadDate = existentMinimalStream.textualUploadDate
|
||||
newerStream.isUploadDateApproximation = existentMinimalStream.isUploadDateApproximation
|
||||
}
|
||||
|
||||
if (existentMinimalStream.duration > 0 && newerStream.duration < 0) {
|
||||
newerStream.duration = existentMinimalStream.duration
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -105,12 +114,18 @@ abstract class StreamDAO : BasicDAO<StreamEntity> {
|
|||
@ColumnInfo(name = STREAM_ID)
|
||||
var uid: Long = 0,
|
||||
|
||||
@field:ColumnInfo(name = StreamEntity.STREAM_TYPE)
|
||||
@ColumnInfo(name = StreamEntity.STREAM_TYPE)
|
||||
var streamType: StreamType,
|
||||
|
||||
@field:ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE)
|
||||
@ColumnInfo(name = StreamEntity.STREAM_TEXTUAL_UPLOAD_DATE)
|
||||
var textualUploadDate: String? = null,
|
||||
|
||||
@field:ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE)
|
||||
var uploadDate: Date? = null)
|
||||
@ColumnInfo(name = StreamEntity.STREAM_UPLOAD_DATE)
|
||||
var uploadDate: Date? = null,
|
||||
|
||||
@ColumnInfo(name = StreamEntity.STREAM_IS_UPLOAD_DATE_APPROXIMATION)
|
||||
var isUploadDateApproximation: Boolean? = null,
|
||||
|
||||
@ColumnInfo(name = StreamEntity.STREAM_DURATION)
|
||||
var duration: Long)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,10 @@ data class StreamEntity(
|
|||
var textualUploadDate: String? = null,
|
||||
|
||||
@ColumnInfo(name = STREAM_UPLOAD_DATE)
|
||||
var uploadDate: Date? = null
|
||||
var uploadDate: Date? = null,
|
||||
|
||||
@ColumnInfo(name = STREAM_IS_UPLOAD_DATE_APPROXIMATION)
|
||||
var isUploadDateApproximation: Boolean? = null
|
||||
) : Serializable {
|
||||
|
||||
@Ignore
|
||||
|
|
@ -58,7 +61,8 @@ data class StreamEntity(
|
|||
serviceId = item.serviceId, url = item.url, title = item.name,
|
||||
streamType = item.streamType, duration = item.duration, uploader = item.uploaderName,
|
||||
thumbnailUrl = item.thumbnailUrl, viewCount = item.viewCount,
|
||||
textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time
|
||||
textualUploadDate = item.textualUploadDate, uploadDate = item.uploadDate?.date()?.time,
|
||||
isUploadDateApproximation = item.uploadDate?.isApproximation
|
||||
)
|
||||
|
||||
@Ignore
|
||||
|
|
@ -66,7 +70,8 @@ data class StreamEntity(
|
|||
serviceId = info.serviceId, url = info.url, title = info.name,
|
||||
streamType = info.streamType, duration = info.duration, uploader = info.uploaderName,
|
||||
thumbnailUrl = info.thumbnailUrl, viewCount = info.viewCount,
|
||||
textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time
|
||||
textualUploadDate = info.textualUploadDate, uploadDate = info.uploadDate?.date()?.time,
|
||||
isUploadDateApproximation = info.uploadDate?.isApproximation
|
||||
)
|
||||
|
||||
@Ignore
|
||||
|
|
@ -84,7 +89,9 @@ data class StreamEntity(
|
|||
|
||||
if (viewCount != null) item.viewCount = viewCount as Long
|
||||
item.textualUploadDate = textualUploadDate
|
||||
item.uploadDate = uploadDate?.let { DateWrapper(Calendar.getInstance().apply { time = it }) }
|
||||
item.uploadDate = uploadDate?.let {
|
||||
DateWrapper(Calendar.getInstance().apply { time = it }, isUploadDateApproximation ?: false)
|
||||
}
|
||||
|
||||
return item
|
||||
}
|
||||
|
|
@ -103,5 +110,6 @@ data class StreamEntity(
|
|||
const val STREAM_VIEWS = "view_count"
|
||||
const val STREAM_TEXTUAL_UPLOAD_DATE = "textual_upload_date"
|
||||
const val STREAM_UPLOAD_DATE = "upload_date"
|
||||
const val STREAM_IS_UPLOAD_DATE_APPROXIMATION = "is_upload_date_approximation"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue