main commit

Post-processing infrastructure
* remove interfaces with one implementation
* fix download resources with unknow length
* marquee style for ProgressDrawable
* "view details" option in mission context menu
* notification for finished downloads
* postprocessing infrastructure: sub-missions, circular file, layers for layers of abstractions for Java IO streams
* Mp4 muxing (only DASH brand)
* WebM muxing
* Captions downloading
* alert dialog for overwrite existing downloads finished or not.

Misc changes
* delete SQLiteDownloadDataSource.java
* delete DownloadMissionSQLiteHelper.java
* implement Localization from #114

Misc fixes (this branch)
* restore old mission listeners variables. Prevents registered listeners get de-referenced on low-end devices
* DownloadManagerService.checkForRunningMission() now return false if the mission has a error.
* use Intent.FLAG_ACTIVITY_NEW_TASK when launching an activity from gigaget threads (apparently it is required in old versions of android)

More changes
* proper error handling "infrastructure"
* queue instead of multiple downloads
* move serialized pending downloads (.giga files) to app data
* stop downloads when swicthing to mobile network (never works, see 2nd point)
* save the thread count for next downloads
* a lot of incoherences fixed
* delete DownloadManagerTest.java (too many changes to keep this file updated)
This commit is contained in:
kapodamy 2018-09-23 15:12:23 -03:00
parent 45fea983b9
commit 5825843f68
48 changed files with 4379 additions and 1119 deletions

View file

@ -0,0 +1,31 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.Mp4DashWriter;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
/**
* @author kapodamy
*/
class Mp4DashMuxer extends Postprocessing {
Mp4DashMuxer(DownloadMission mission) {
super(mission);
recommendedReserve = 2048 * 1024;// 2 MiB
worksOnSameFile = true;
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
Mp4DashWriter muxer = new Mp4DashWriter(sources);
muxer.parseSources();
muxer.selectTracks(0, 0);
muxer.build(out);
return OK_RESULT;
}
}

View file

@ -0,0 +1,149 @@
package us.shandian.giga.postprocessing;
import android.os.Message;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.File;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.postprocessing.io.ChunkFileInputStream;
import us.shandian.giga.postprocessing.io.CircularFile;
import us.shandian.giga.service.DownloadManagerService;
public abstract class Postprocessing {
static final byte OK_RESULT = DownloadMission.ERROR_NOTHING;
public static final String ALGORITHM_TTML_CONVERTER = "ttml";
public static final String ALGORITHM_MP4_DASH_MUXER = "mp4D";
public static final String ALGORITHM_WEBM_MUXER = "webm";
private static final String ALGORITHM_TEST_ALGO = "test";
public static Postprocessing getAlgorithm(String algorithmName, DownloadMission mission) {
if (null == algorithmName) {
throw new NullPointerException("algorithmName");
} else switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER:
return new TttmlConverter(mission);
case ALGORITHM_MP4_DASH_MUXER:
return new Mp4DashMuxer(mission);
case ALGORITHM_WEBM_MUXER:
return new WebMMuxer(mission);
case ALGORITHM_TEST_ALGO:
return new TestAlgo(mission);
/*case "example-algorithm":
return new ExampleAlgorithm(mission);*/
default:
throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName);
}
}
/**
* Get a boolean value that indicate if the given algorithm work on the same
* file
*/
public boolean worksOnSameFile;
/**
* Get the recommended space to reserve for the given algorithm. The amount
* is in bytes
*/
public int recommendedReserve;
protected DownloadMission mission;
Postprocessing(DownloadMission mission) {
this.mission = mission;
}
public void run() throws IOException {
File file = mission.getDownloadedFile();
CircularFile out = null;
ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
try {
int i = 0;
for (; i < sources.length - 1; i++) {
sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.offsets[i + 1], "rw");
}
sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw");
int[] idx = {0};
CircularFile.OffsetChecker checker = () -> {
while (idx[0] < sources.length) {
/*
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
* or the CircularFile can lead to unexpected results
*/
if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) {
idx[0]++;
continue;// the selected source is not used anymore
}
return sources[idx[0]].getFilePointer() - 1;
}
return -1;
};
out = new CircularFile(file, 0, this::progressReport, checker);
mission.done = 0;
int result = process(out, sources);
if (result == OK_RESULT) {
long finalLength = out.finalizeFile();
mission.done = finalLength;
mission.length = finalLength;
} else {
mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION;
mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
}
if (result != OK_RESULT && worksOnSameFile) {
//noinspection ResultOfMethodCallIgnored
new File(mission.location, mission.name).delete();
}
} finally {
for (SharpStream source : sources) {
if (source != null && !source.isDisposed()) {
source.dispose();
}
}
if (out != null) {
out.dispose();
}
}
}
/**
* Abstract method to execute the pos-processing algorithm
*
* @param out output stream
* @param sources files to be processed
* @return a error code, 0 means the operation was successful
* @throws IOException if an I/O error occurs.
*/
abstract int process(SharpStream out, SharpStream... sources) throws IOException;
String getArgumentAt(int index, String defaultValue) {
if (mission.postprocessingArgs == null || index >= mission.postprocessingArgs.length) {
return defaultValue;
}
return mission.postprocessingArgs[index];
}
private void progressReport(long done) {
mission.done = done;
if (mission.length < mission.done) mission.length = mission.done;
Message m = new Message();
m.what = DownloadManagerService.MESSAGE_PROGRESS;
m.obj = mission;
mission.mHandler.sendMessage(m);
}
}

View file

@ -0,0 +1,54 @@
package us.shandian.giga.postprocessing;
import android.util.Log;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.IOException;
import java.util.Random;
import us.shandian.giga.get.DownloadMission;
/**
* Algorithm for testing proposes
*/
class TestAlgo extends Postprocessing {
public TestAlgo(DownloadMission mission) {
super(mission);
worksOnSameFile = true;
recommendedReserve = 4096 * 1024;// 4 KiB
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
int written = 0;
int size = 5 * 1024 * 1024;// 5 MiB
byte[] buffer = new byte[8 * 1024];//8 KiB
mission.length = size;
Random rnd = new Random();
// only write random data
sources[0].dispose();
while (written < size) {
rnd.nextBytes(buffer);
int read = Math.min(buffer.length, size - written);
out.write(buffer, 0, read);
try {
Thread.sleep((int) (Math.random() * 10));
} catch (InterruptedException e) {
return -1;
}
written += read;
}
return Postprocessing.OK_RESULT;
}
}

View file

@ -0,0 +1,49 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import org.schabi.newpipe.extractor.utils.SubtitleConverter;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.postprocessing.io.SharpInputStream;
/**
* @author kapodamy
*/
class TttmlConverter extends Postprocessing {
TttmlConverter(DownloadMission mission) {
super(mission);
recommendedReserve = 0;// due how XmlPullParser works, the xml is fully loaded on the ram
worksOnSameFile = true;
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
// check if the subtitle is already in srt and copy, this should never happen
String format = getArgumentAt(0, null);
if (format == null || format.equals("ttml")) {
SubtitleConverter ttmlDumper = new SubtitleConverter();
int res = ttmlDumper.dumpTTML(
sources[0],
out,
getArgumentAt(1, "true").equals("true"),
getArgumentAt(2, "true").equals("true")
);
return res == 0 ? OK_RESULT : res;
} else if (format.equals("srt")) {
byte[] buffer = new byte[8 * 1024];
int read;
while ((read = sources[0].read(buffer)) > 0) {
out.write(buffer, 0, read);
}
return OK_RESULT;
}
throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format);
}
}

View file

@ -0,0 +1,44 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.WebMReader.TrackKind;
import org.schabi.newpipe.extractor.utils.WebMReader.WebMTrack;
import org.schabi.newpipe.extractor.utils.WebMWriter;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
/**
* @author kapodamy
*/
class WebMMuxer extends Postprocessing {
WebMMuxer(DownloadMission mission) {
super(mission);
recommendedReserve = (1024 + 512) * 1024;// 1.50 MiB
worksOnSameFile = true;
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
WebMWriter muxer = new WebMWriter(sources);
muxer.parseSources();
// youtube uses a webm with a fake video track that acts as a "cover image"
WebMTrack[] tracks = muxer.getTracksFromSource(1);
int audioTrackIndex = 0;
for (int i = 0; i < tracks.length; i++) {
if (tracks[i].kind == TrackKind.Audio) {
audioTrackIndex = i;
break;
}
}
muxer.selectTracks(0, audioTrackIndex);
muxer.build(out);
return OK_RESULT;
}
}

View file

@ -0,0 +1,153 @@
package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
public class ChunkFileInputStream extends SharpStream {
private RandomAccessFile source;
private final long offset;
private final long length;
private long position;
public ChunkFileInputStream(File file, long start, long end, String mode) throws IOException {
source = new RandomAccessFile(file, mode);
offset = start;
length = end - start;
position = 0;
if (length < 1) {
source.close();
throw new IOException("The chunk is empty or invalid");
}
if (source.length() < end) {
try {
throw new IOException(String.format("invalid file length. expected = %s found = %s", end, source.length()));
} finally {
source.close();
}
}
source.seek(offset);
}
/**
* Get absolute position on file
*
* @return the position
*/
public long getFilePointer() {
return offset + position;
}
@Override
public int read() throws IOException {
if ((position + 1) > length) {
return 0;
}
int res = source.read();
if (res >= 0) {
position++;
}
return res;
}
@Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte b[], int off, int len) throws IOException {
if ((position + len) > length) {
len = (int) (length - position);
}
if (len == 0) {
return 0;
}
int res = source.read(b, off, len);
position += res;
return res;
}
@Override
public long skip(long pos) throws IOException {
pos = Math.min(pos + position, length);
if (pos == 0) {
return 0;
}
source.seek(offset + pos);
long oldPos = position;
position = pos;
return pos - oldPos;
}
@Override
public int available() {
return (int) (length - position);
}
@SuppressWarnings("EmptyCatchBlock")
@Override
public void dispose() {
try {
source.close();
} catch (IOException err) {
} finally {
source = null;
}
}
@Override
public boolean isDisposed() {
return source == null;
}
@Override
public void rewind() throws IOException {
position = 0;
source.seek(offset);
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canRead() {
return true;
}
@Override
public boolean canWrite() {
return false;
}
@Override
public void write(byte value) {
}
@Override
public void write(byte[] buffer) {
}
@Override
public void write(byte[] buffer, int offset, int count) {
}
@Override
public void flush() {
}
}

View file

@ -0,0 +1,345 @@
package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
public class CircularFile extends SharpStream {
private final static int AUX_BUFFER_SIZE = 1024 * 1024;// 1 MiB
private final static int AUX2_BUFFER_SIZE = 256 * 1024;// 256 KiB
private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB
private RandomAccessFile out;
private long position;
private long maxLengthKnown = -1;
private ArrayList<ManagedBuffer> auxiliaryBuffers;
private OffsetChecker callback;
private ManagedBuffer queue;
private long startOffset;
private ProgressReport onProgress;
private long reportPosition;
public CircularFile(File file, long offset, ProgressReport progressReport, OffsetChecker checker) throws IOException {
if (checker == null) {
throw new NullPointerException("checker is null");
}
try {
queue = new ManagedBuffer(QUEUE_BUFFER_SIZE);
out = new RandomAccessFile(file, "rw");
out.seek(offset);
position = offset;
} catch (IOException err) {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
// nothing to do
}
throw err;
}
auxiliaryBuffers = new ArrayList<>(1);
callback = checker;
startOffset = offset;
reportPosition = offset;
onProgress = progressReport;
}
/**
* Close the file without flushing any buffer
*/
@Override
public void dispose() {
try {
auxiliaryBuffers = null;
if (out != null) {
out.close();
out = null;
}
} catch (IOException err) {
// nothing to do
}
}
/**
* Flush any buffer and close the output file. Use this method if the
* operation is successful
*
* @return the final length of the file
* @throws IOException if an I/O error occurs
*/
public long finalizeFile() throws IOException {
flushEverything();
if (maxLengthKnown > -1) {
position = maxLengthKnown;
}
if (position < out.length()) {
out.setLength(position);
}
dispose();
return position;
}
@Override
public void write(byte b) throws IOException {
write(new byte[]{b}, 0, 1);
}
@Override
public void write(byte b[]) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte b[], int off, int len) throws IOException {
if (len == 0) {
return;
}
long end = callback.check();
int available;
if (end == -1) {
available = Integer.MAX_VALUE;
} else {
if (end < startOffset) {
throw new IOException("The reported offset is invalid. reported offset is " + String.valueOf(end));
}
available = (int) (end - position);
}
while (available > 0 && auxiliaryBuffers.size() > 0) {
ManagedBuffer aux = auxiliaryBuffers.get(0);
if ((queue.size + aux.size) > available) {
available = 0;// wait for next check
break;
}
writeQueue(aux.buffer, 0, aux.size);
available -= aux.size;
aux.dereference();
auxiliaryBuffers.remove(0);
}
if (available > (len + queue.size)) {
writeQueue(b, off, len);
} else {
int i = auxiliaryBuffers.size() - 1;
while (len > 0) {
if (i < 0) {
// allocate a new auxiliary buffer
auxiliaryBuffers.add(new ManagedBuffer(AUX_BUFFER_SIZE));
i++;
}
ManagedBuffer aux = auxiliaryBuffers.get(i);
available = aux.available();
if (available < 1) {
// secondary auxiliary buffer
available = len;
aux = new ManagedBuffer(Math.max(len, AUX2_BUFFER_SIZE));
auxiliaryBuffers.add(aux);
i++;
} else {
available = Math.min(len, available);
}
aux.write(b, off, available);
len -= available;
if (len < 1) {
break;
}
off += available;
}
}
}
private void writeOutside(byte buffer[], int offset, int length) throws IOException {
out.write(buffer, offset, length);
position += length;
if (onProgress != null && position > reportPosition) {
reportPosition = position + AUX2_BUFFER_SIZE;// notify every 256 KiB (approx)
onProgress.report(position);
}
}
private void writeQueue(byte[] buffer, int offset, int length) throws IOException {
while (length > 0) {
if (queue.available() < length) {
flushQueue();
if (length >= queue.buffer.length) {
writeOutside(buffer, offset, length);
return;
}
}
int size = Math.min(queue.available(), length);
queue.write(buffer, offset, size);
offset += size;
length -= size;
}
}
private void flushQueue() throws IOException {
writeOutside(queue.buffer, 0, queue.size);
queue.size = 0;
}
private void flushEverything() throws IOException {
flushQueue();
if (auxiliaryBuffers.size() > 0) {
for (ManagedBuffer aux : auxiliaryBuffers) {
writeOutside(aux.buffer, 0, aux.size);
aux.dereference();
}
auxiliaryBuffers.clear();
}
}
/**
* Flush any buffer directly to the file. Warning: use this method ONLY if
* all read dependencies are disposed
*
* @throws IOException if the dependencies are not disposed
*/
@Override
public void flush() throws IOException {
if (callback.check() != -1) {
throw new IOException("All read dependencies of this file must be disposed first");
}
flushEverything();
// Save the current file length in case the method {@code rewind()} is called
if (position > maxLengthKnown) {
maxLengthKnown = position;
}
}
@Override
public void rewind() throws IOException {
flush();
out.seek(startOffset);
if (onProgress != null) onProgress.report(-position);
position = startOffset;
reportPosition = startOffset;
}
@Override
public long skip(long amount) throws IOException {
flush();
position += amount;
out.seek(position);
return amount;
}
@Override
public boolean isDisposed() {
return out == null;
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canWrite() {
return true;
}
//<editor-fold defaultState="collapsed" desc="stub read methods">
@Override
public boolean canRead() {
return false;
}
@Override
public int read() {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer) {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer, int offset, int count) {
throw new UnsupportedOperationException("write-only");
}
@Override
public int available() {
throw new UnsupportedOperationException("write-only");
}
//</editor-fold>
public interface OffsetChecker {
/**
* Checks the amount of available space ahead
*
* @return absolute offset in the file where no more data SHOULD NOT be
* written. If the value is -1 the whole file will be used
*/
long check();
}
public interface ProgressReport {
void report(long progress);
}
class ManagedBuffer {
byte[] buffer;
int size;
ManagedBuffer(int length) {
buffer = new byte[length];
}
void dereference() {
buffer = null;
size = 0;
}
protected int available() {
return buffer.length - size;
}
private void write(byte[] b, int off, int len) {
System.arraycopy(b, off, buffer, size, len);
size += len;
}
@Override
public String toString() {
return "holding: " + String.valueOf(size) + " length: " + String.valueOf(buffer.length) + " available: " + String.valueOf(available());
}
}
}

View file

@ -0,0 +1,126 @@
package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
/**
* @author kapodamy
*/
public class FileStream extends SharpStream {
public enum Mode {
Read,
ReadWrite
}
public RandomAccessFile source;
private final Mode mode;
public FileStream(String path, Mode mode) throws IOException {
String flags;
if (mode == Mode.Read) {
flags = "r";
} else {
flags = "rw";
}
this.mode = mode;
source = new RandomAccessFile(path, flags);
}
@Override
public int read() throws IOException {
return source.read();
}
@Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte b[], int off, int len) throws IOException {
return source.read(b, off, len);
}
@Override
public long skip(long pos) throws IOException {
FileChannel fc = source.getChannel();
fc.position(fc.position() + pos);
return pos;
}
@Override
public int available() {
try {
return (int) (source.length() - source.getFilePointer());
} catch (IOException ex) {
return 0;
}
}
@SuppressWarnings("EmptyCatchBlock")
@Override
public void dispose() {
try {
source.close();
} catch (IOException err) {
} finally {
source = null;
}
}
@Override
public boolean isDisposed() {
return source == null;
}
@Override
public void rewind() throws IOException {
source.getChannel().position(0);
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canRead() {
return mode == Mode.Read || mode == Mode.ReadWrite;
}
@Override
public boolean canWrite() {
return mode == Mode.ReadWrite;
}
@Override
public void write(byte value) throws IOException {
source.write(value);
}
@Override
public void write(byte[] buffer) throws IOException {
source.write(buffer);
}
@Override
public void write(byte[] buffer, int offset, int count) throws IOException {
source.write(buffer, offset, count);
}
@Override
public void flush() {
}
@Override
public void setLength(long length) throws IOException {
source.setLength(length);
}
}

View file

@ -0,0 +1,59 @@
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package us.shandian.giga.postprocessing.io;
import android.support.annotation.NonNull;
import org.schabi.newpipe.extractor.utils.io.SharpStream;
import java.io.IOException;
import java.io.InputStream;
/**
* Wrapper for the classic {@link java.io.InputStream}
* @author kapodamy
*/
public class SharpInputStream extends InputStream {
private final SharpStream base;
public SharpInputStream(SharpStream base) throws IOException {
if (!base.canRead()) {
throw new IOException("The provided stream is not readable");
}
this.base = base;
}
@Override
public int read() throws IOException {
return base.read();
}
@Override
public int read(@NonNull byte[] bytes) throws IOException {
return base.read(bytes);
}
@Override
public int read(@NonNull byte[] bytes, int i, int i1) throws IOException {
return base.read(bytes, i, i1);
}
@Override
public long skip(long l) throws IOException {
return base.skip(l);
}
@Override
public int available() {
return base.available();
}
@Override
public void close() {
base.dispose();
}
}