main commit

Post-processing infrastructure
* remove interfaces with one implementation
* fix download resources with unknow length
* marquee style for ProgressDrawable
* "view details" option in mission context menu
* notification for finished downloads
* postprocessing infrastructure: sub-missions, circular file, layers for layers of abstractions for Java IO streams
* Mp4 muxing (only DASH brand)
* WebM muxing
* Captions downloading
* alert dialog for overwrite existing downloads finished or not.

Misc changes
* delete SQLiteDownloadDataSource.java
* delete DownloadMissionSQLiteHelper.java
* implement Localization from #114

Misc fixes (this branch)
* restore old mission listeners variables. Prevents registered listeners get de-referenced on low-end devices
* DownloadManagerService.checkForRunningMission() now return false if the mission has a error.
* use Intent.FLAG_ACTIVITY_NEW_TASK when launching an activity from gigaget threads (apparently it is required in old versions of android)

More changes
* proper error handling "infrastructure"
* queue instead of multiple downloads
* move serialized pending downloads (.giga files) to app data
* stop downloads when swicthing to mobile network (never works, see 2nd point)
* save the thread count for next downloads
* a lot of incoherences fixed
* delete DownloadManagerTest.java (too many changes to keep this file updated)
This commit is contained in:
kapodamy 2018-09-23 15:12:23 -03:00
parent 2dc2a56671
commit 21e205a6be
48 changed files with 4379 additions and 1119 deletions

View file

@ -0,0 +1,158 @@
package us.shandian.giga.get;
import android.support.annotation.NonNull;
import android.util.Log;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.nio.channels.ClosedByInterruptException;
import us.shandian.giga.util.Utility;
import static org.schabi.newpipe.BuildConfig.DEBUG;
public class DownloadInitializer implements Runnable {
private final static String TAG = "DownloadInitializer";
final static int mId = 0;
private DownloadMission mMission;
DownloadInitializer(@NonNull DownloadMission mission) {
mMission = mission;
}
@Override
public void run() {
if (mMission.current > 0) mMission.resetState();
int retryCount = 0;
while (true) {
try {
mMission.currentThreadCount = mMission.threadCount;
HttpURLConnection conn = mMission.openConnection(mId, -1, -1);
if (!mMission.running || Thread.interrupted()) return;
mMission.length = conn.getContentLength();
if (mMission.length == 0) {
mMission.notifyError(DownloadMission.ERROR_HTTP_NO_CONTENT, null);
return;
}
// check for dynamic generated content
if (mMission.length == -1 && conn.getResponseCode() == 200) {
mMission.blocks = 0;
mMission.length = 0;
mMission.fallback = true;
mMission.unknownLength = true;
mMission.currentThreadCount = 1;
if (DEBUG) {
Log.d(TAG, "falling back (unknown length)");
}
} else {
// Open again
conn = mMission.openConnection(mId, mMission.length - 10, mMission.length);
int code = conn.getResponseCode();
if (!mMission.running || Thread.interrupted()) return;
if (code == 206) {
if (mMission.currentThreadCount > 1) {
mMission.blocks = mMission.length / DownloadMission.BLOCK_SIZE;
if (mMission.currentThreadCount > mMission.blocks) {
mMission.currentThreadCount = (int) mMission.blocks;
}
if (mMission.currentThreadCount <= 0) {
mMission.currentThreadCount = 1;
}
if (mMission.blocks * DownloadMission.BLOCK_SIZE < mMission.length) {
mMission.blocks++;
}
} else {
// if one thread is solicited don't calculate blocks, is useless
mMission.blocks = 0;
mMission.fallback = true;
mMission.unknownLength = false;
}
if (DEBUG) {
Log.d(TAG, "http response code = " + code);
}
} else {
// Fallback to single thread
mMission.blocks = 0;
mMission.fallback = true;
mMission.unknownLength = false;
mMission.currentThreadCount = 1;
if (DEBUG) {
Log.d(TAG, "falling back due http response code = " + code);
}
}
}
for (long i = 0; i < mMission.currentThreadCount; i++) {
mMission.threadBlockPositions.add(i);
mMission.threadBytePositions.add(0);
}
File file;
if (mMission.current == 0) {
file = new File(mMission.location);
if (!Utility.mkdir(file, true)) {
mMission.notifyError(DownloadMission.ERROR_PATH_CREATION, null);
return;
}
file = new File(file, mMission.name);
// if the name is used by "something", delete it
if (file.exists() && !file.isFile() && !file.delete()) {
mMission.notifyError(DownloadMission.ERROR_FILE_CREATION, null);
return;
}
if (!file.exists() && !file.createNewFile()) {
mMission.notifyError(DownloadMission.ERROR_FILE_CREATION, null);
return;
}
} else {
file = new File(mMission.location, mMission.name);
}
RandomAccessFile af = new RandomAccessFile(file, "rw");
af.setLength(mMission.offsets[mMission.current] + mMission.length);
af.seek(mMission.offsets[mMission.current]);
af.close();
if (Thread.interrupted()) return;
mMission.running = false;
break;
} catch (Exception e) {
if (e instanceof ClosedByInterruptException) {
return;
} else if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
return;
}
if (retryCount++ > mMission.maxRetry) {
Log.e(TAG, "initializer failed", e);
mMission.running = false;
mMission.notifyError(e);
return;
}
//try again
Log.e(TAG, "initializer failed, retrying", e);
}
}
mMission.start();
}
}

View file

@ -1,53 +0,0 @@
package us.shandian.giga.get;
public interface DownloadManager {
int BLOCK_SIZE = 512 * 1024;
/**
* Start a new download mission
*
* @param url the url to download
* @param location the location
* @param name the name of the file to create
* @param isAudio true if the download is an audio file
* @param threads the number of threads maximal used to download chunks of the file. @return the identifier of the mission.
*/
int startMission(String url, String location, String name, boolean isAudio, int threads);
/**
* Resume the execution of a download mission.
*
* @param id the identifier of the mission to resume.
*/
void resumeMission(int id);
/**
* Pause the execution of a download mission.
*
* @param id the identifier of the mission to pause.
*/
void pauseMission(int id);
/**
* Deletes the mission from the downloaded list but keeps the downloaded file.
*
* @param id The mission identifier
*/
void deleteMission(int id);
/**
* Get the download mission by its identifier
*
* @param id the identifier of the download mission
* @return the download mission or null if the mission doesn't exist
*/
DownloadMission getMission(int id);
/**
* Get the number of download missions.
*
* @return the number of download missions.
*/
int getCount();
}

View file

@ -1,102 +1,165 @@
package us.shandian.giga.get;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import java.io.File;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.lang.ref.WeakReference;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLException;
import us.shandian.giga.postprocessing.Postprocessing;
import us.shandian.giga.service.DownloadManagerService;
import us.shandian.giga.util.Utility;
import static org.schabi.newpipe.BuildConfig.DEBUG;
public class DownloadMission implements Serializable {
private static final long serialVersionUID = 0L;
public class DownloadMission extends Mission {
private static final long serialVersionUID = 3L;// last bump: 16 october 2018
private static final String TAG = DownloadMission.class.getSimpleName();
static final int BUFFER_SIZE = 64 * 1024;
final static int BLOCK_SIZE = 512 * 1024;
public interface MissionListener {
HashMap<MissionListener, Handler> handlerStore = new HashMap<>();
private static final String TAG = "DownloadMission";
void onProgressUpdate(DownloadMission downloadMission, long done, long total);
void onFinish(DownloadMission downloadMission);
void onError(DownloadMission downloadMission, int errCode);
}
public static final int ERROR_SERVER_UNSUPPORTED = 206;
public static final int ERROR_UNKNOWN = 233;
public static final int ERROR_NOTHING = -1;
public static final int ERROR_PATH_CREATION = 1000;
public static final int ERROR_FILE_CREATION = 1001;
public static final int ERROR_UNKNOWN_EXCEPTION = 1002;
public static final int ERROR_PERMISSION_DENIED = 1003;
public static final int ERROR_SSL_EXCEPTION = 1004;
public static final int ERROR_UNKNOWN_HOST = 1005;
public static final int ERROR_CONNECT_HOST = 1006;
public static final int ERROR_POSTPROCESSING_FAILED = 1007;
public static final int ERROR_HTTP_NO_CONTENT = 204;
public static final int ERROR_HTTP_UNSUPPORTED_RANGE = 206;
/**
* The filename
* The urls of the file to download
*/
public String name;
public String[] urls;
/**
* The url of the file to download
* Number of blocks the size of {@link DownloadMission#BLOCK_SIZE}
*/
public String url;
/**
* The directory to store the download
*/
public String location;
/**
* Number of blocks the size of {@link DownloadManager#BLOCK_SIZE}
*/
public long blocks;
/**
* Number of bytes
*/
public long length;
long blocks = -1;
/**
* Number of bytes downloaded
*/
public long done;
/**
* Indicates a file generated dynamically on the web server
*/
public boolean unknownLength;
/**
* offset in the file where the data should be written
*/
public long[] offsets;
/**
* The post-processing algorithm arguments
*/
public String[] postprocessingArgs;
/**
* The post-processing algorithm name
*/
public String postprocessingName;
/**
* Indicates if the post-processing algorithm is actually running, used to detect corrupt downloads
*/
public boolean postprocessingRunning;
/**
* Indicate if the post-processing algorithm works on the same file
*/
public boolean postprocessingThis;
/**
* The current resource to download {@code urls[current]}
*/
public int current;
/**
* Metadata where the mission state is saved
*/
public File metadata;
/**
* maximum attempts
*/
public int maxRetry;
public int threadCount = 3;
public int finishCount;
private final List<Long> threadPositions = new ArrayList<>();
public final Map<Long, Boolean> blockState = new HashMap<>();
public boolean running;
public boolean finished;
public boolean fallback;
public int errCode = -1;
public long timestamp;
boolean fallback;
private int finishCount;
public transient boolean running;
public transient boolean enqueued = true;
public int errCode = ERROR_NOTHING;
public transient Exception errObject = null;
public transient boolean recovered;
private transient ArrayList<WeakReference<MissionListener>> mListeners = new ArrayList<>();
public transient Handler mHandler;
private transient boolean mWritingToFile;
private static final int NO_IDENTIFIER = -1;
@SuppressWarnings("UseSparseArrays")// LongSparseArray is not serializable
private final HashMap<Long, Boolean> blockState = new HashMap<>();
final List<Long> threadBlockPositions = new ArrayList<>();
final List<Integer> threadBytePositions = new ArrayList<>();
private transient boolean deleted;
int currentThreadCount;
private transient Thread[] threads = null;
private transient Thread init = null;
protected DownloadMission() {
public DownloadMission() {
}
public DownloadMission(String name, String url, String location) {
public DownloadMission(String url, String name, String location, char kind) {
this(new String[]{url}, name, location, kind, null, null);
}
public DownloadMission(String[] urls, String name, String location, char kind, String postprocessingName, String[] postprocessingArgs) {
if (name == null) throw new NullPointerException("name is null");
if (name.isEmpty()) throw new IllegalArgumentException("name is empty");
if (url == null) throw new NullPointerException("url is null");
if (url.isEmpty()) throw new IllegalArgumentException("url is empty");
if (urls == null) throw new NullPointerException("urls is null");
if (urls.length < 1) throw new IllegalArgumentException("urls is empty");
if (location == null) throw new NullPointerException("location is null");
if (location.isEmpty()) throw new IllegalArgumentException("location is empty");
this.url = url;
this.urls = urls;
this.name = name;
this.location = location;
}
this.kind = kind;
this.offsets = new long[urls.length];
if (postprocessingName != null) {
Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, null);
this.postprocessingThis = algorithm.worksOnSameFile;
this.offsets[0] = algorithm.recommendedReserve;
this.postprocessingName = postprocessingName;
this.postprocessingArgs = postprocessingArgs;
} else {
if (DEBUG && urls.length > 1) {
Log.w(TAG, "mission created with multiple urls ¿missing post-processing algorithm?");
}
}
}
private void checkBlock(long block) {
if (block < 0 || block >= blocks) {
@ -110,12 +173,12 @@ public class DownloadMission implements Serializable {
* @param block the block identifier
* @return true if the block is reserved and false if otherwise
*/
public boolean isBlockPreserved(long block) {
boolean isBlockPreserved(long block) {
checkBlock(block);
return blockState.containsKey(block) ? blockState.get(block) : false;
}
public void preserveBlock(long block) {
void preserveBlock(long block) {
checkBlock(block);
synchronized (blockState) {
blockState.put(block, true);
@ -123,125 +186,192 @@ public class DownloadMission implements Serializable {
}
/**
* Set the download position of the file
* Set the block of the file
*
* @param threadId the identifier of the thread
* @param position the download position of the thread
* @param position the block of the thread
*/
public void setPosition(int threadId, long position) {
threadPositions.set(threadId, position);
void setBlockPosition(int threadId, long position) {
threadBlockPositions.set(threadId, position);
}
/**
* Get the position of a thread
* Get the block of a file
*
* @param threadId the identifier of the thread
* @return the position for the thread
* @return the block for the thread
*/
public long getPosition(int threadId) {
return threadPositions.get(threadId);
long getBlockPosition(int threadId) {
return threadBlockPositions.get(threadId);
}
public synchronized void notifyProgress(long deltaLen) {
/**
* Save the position of the desired thread
*
* @param threadId the identifier of the thread
* @param position the relative position in bytes or zero
*/
void setThreadBytePosition(int threadId, int position) {
threadBytePositions.set(threadId, position);
}
/**
* Get position inside of the block, where thread will be resumed
*
* @param threadId the identifier of the thread
* @return the relative position in bytes or zero
*/
int getBlockBytePosition(int threadId) {
return threadBytePositions.get(threadId);
}
/**
* Open connection
*
* @param threadId id of the calling thread, used only for debug
* @param rangeStart range start
* @param rangeEnd range end
* @return a {@link java.net.URLConnection URLConnection} linking to the URL.
* @throws IOException if an I/O exception occurs.
* @throws HttpError if the the http response is not satisfiable
*/
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException, HttpError {
URL url = new URL(urls[current]);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setInstanceFollowRedirects(true);
if (rangeStart >= 0) {
String req = "bytes=" + rangeStart + "-";
if (rangeEnd > 0) req += rangeEnd;
conn.setRequestProperty("Range", req);
if (DEBUG) {
Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range"));
Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + conn.getResponseCode());
}
}
int statusCode = conn.getResponseCode();
switch (statusCode) {
case 204:
case 205:
case 207:
throw new HttpError(conn.getResponseCode());
default:
if (statusCode < 200 || statusCode > 299) {
throw new HttpError(statusCode);
}
}
return conn;
}
private void notify(int what) {
Message m = new Message();
m.what = what;
m.obj = this;
mHandler.sendMessage(m);
}
synchronized void notifyProgress(long deltaLen) {
if (!running) return;
if (recovered) {
recovered = false;
}
if (unknownLength) {
length += deltaLen;// Update length before proceeding
}
done += deltaLen;
if (done > length) {
done = length;
}
if (done != length) {
writeThisToFile();
if (done != length && !deleted && !mWritingToFile) {
mWritingToFile = true;
runAsync(-2, this::writeThisToFile);
}
for (WeakReference<MissionListener> ref : mListeners) {
final MissionListener listener = ref.get();
if (listener != null) {
MissionListener.handlerStore.get(listener).post(new Runnable() {
@Override
public void run() {
listener.onProgressUpdate(DownloadMission.this, done, length);
}
});
}
notify(DownloadManagerService.MESSAGE_PROGRESS);
}
synchronized void notifyError(Exception err) {
Log.e(TAG, "notifyError()", err);
if (err instanceof FileNotFoundException) {
notifyError(ERROR_FILE_CREATION, null);
} else if (err instanceof SSLException) {
notifyError(ERROR_SSL_EXCEPTION, null);
} else if (err instanceof HttpError) {
notifyError(((HttpError) err).statusCode, null);
} else if (err instanceof ConnectException) {
notifyError(ERROR_CONNECT_HOST, null);
} else if (err instanceof UnknownHostException) {
notifyError(ERROR_UNKNOWN_HOST, null);
} else {
notifyError(ERROR_UNKNOWN_EXCEPTION, err);
}
}
/**
* Called by a download thread when it finished.
*/
public synchronized void notifyFinished() {
if (errCode > 0) return;
synchronized void notifyError(int code, Exception err) {
Log.e(TAG, "notifyError() code = " + code, err);
errCode = code;
errObject = err;
pause();
notify(DownloadManagerService.MESSAGE_ERROR);
}
synchronized void notifyFinished() {
if (errCode > ERROR_NOTHING) return;
finishCount++;
if (finishCount == threadCount) {
onFinish();
if (finishCount == currentThreadCount) {
if ((current + 1) < urls.length) {
// prepare next sub-mission
long current_offset = offsets[current++];
offsets[current] = current_offset + length;
initializer();
return;
}
current++;
unknownLength = false;
if (!doPostprocessing()) return;
if (errCode > ERROR_NOTHING) return;
if (DEBUG) {
Log.d(TAG, "onFinish");
}
running = false;
deleteThisFromFile();
notify(DownloadManagerService.MESSAGE_FINISHED);
}
}
/**
* Called when all parts are downloaded
*/
private void onFinish() {
if (errCode > 0) return;
private void notifyPostProcessing(boolean processing) {
if (DEBUG) {
Log.d(TAG, "onFinish");
Log.d(TAG, (processing ? "enter" : "exit") + " postprocessing on " + location + File.separator + name);
}
running = false;
finished = true;
deleteThisFromFile();
for (WeakReference<MissionListener> ref : mListeners) {
final MissionListener listener = ref.get();
if (listener != null) {
MissionListener.handlerStore.get(listener).post(new Runnable() {
@Override
public void run() {
listener.onFinish(DownloadMission.this);
}
});
synchronized (blockState) {
if (!processing) {
postprocessingName = null;
postprocessingArgs = null;
}
}
}
public synchronized void notifyError(int err) {
errCode = err;
writeThisToFile();
for (WeakReference<MissionListener> ref : mListeners) {
final MissionListener listener = ref.get();
MissionListener.handlerStore.get(listener).post(new Runnable() {
@Override
public void run() {
listener.onError(DownloadMission.this, errCode);
}
});
}
}
public synchronized void addListener(MissionListener listener) {
Handler handler = new Handler(Looper.getMainLooper());
MissionListener.handlerStore.put(listener, handler);
mListeners.add(new WeakReference<>(listener));
}
public synchronized void removeListener(MissionListener listener) {
for (Iterator<WeakReference<MissionListener>> iterator = mListeners.iterator();
iterator.hasNext(); ) {
WeakReference<MissionListener> weakRef = iterator.next();
if (listener != null && listener == weakRef.get()) {
iterator.remove();
}
// don't return without fully write the current state
postprocessingRunning = processing;
Utility.writeToFile(metadata, DownloadMission.this);
}
}
@ -249,92 +379,206 @@ public class DownloadMission implements Serializable {
* Start downloading with multiple threads.
*/
public void start() {
if (!running && !finished) {
running = true;
if (running || current >= urls.length) return;
enqueued = false;
running = true;
errCode = ERROR_NOTHING;
if (!fallback) {
for (int i = 0; i < threadCount; i++) {
if (threadPositions.size() <= i && !recovered) {
threadPositions.add((long) i);
}
new Thread(new DownloadRunnable(this, i)).start();
}
} else {
// In fallback mode, resuming is not supported.
threadCount = 1;
if (blocks < 0) {
initializer();
return;
}
init = null;
if (threads == null) {
threads = new Thread[currentThreadCount];
}
if (fallback) {
if (unknownLength) {
done = 0;
blocks = 0;
new Thread(new DownloadRunnableFallback(this)).start();
length = 0;
}
threads[0] = runAsync(1, new DownloadRunnableFallback(this));
} else {
for (int i = 0; i < currentThreadCount; i++) {
threads[i] = runAsync(i + 1, new DownloadRunnable(this, i));
}
}
}
public void pause() {
if (running) {
running = false;
recovered = true;
/**
* Pause the mission, does not affect the blocks that are being downloaded.
*/
public synchronized void pause() {
if (!running) return;
// TODO: Notify & Write state to info file
// if (err)
running = false;
recovered = true;
enqueued = false;
if (init != null && init != Thread.currentThread() && init.isAlive()) {
init.interrupt();
try {
init.join();
} catch (InterruptedException e) {
// nothing to do
}
resetState();
return;
}
if (DEBUG && blocks < 1) {
Log.w(TAG, "pausing a download that can not be resumed.");
}
if (threads == null || Thread.interrupted()) {
writeThisToFile();
return;
}
// wait for all threads are suspended before save the state
runAsync(-1, () -> {
try {
for (Thread thread : threads) {
if (thread == Thread.currentThread()) continue;
if (thread.isAlive()) {
thread.interrupt();
thread.join();
}
}
} catch (Exception e) {
// nothing to do
} finally {
writeThisToFile();
}
});
}
/**
* Removes the file and the meta file
*/
public void delete() {
deleteThisFromFile();
new File(location, name).delete();
@Override
public boolean delete() {
deleted = true;
boolean res = deleteThisFromFile();
if (!super.delete()) res = false;
return res;
}
void resetState() {
done = 0;
blocks = -1;
errCode = ERROR_NOTHING;
fallback = false;
unknownLength = false;
finishCount = 0;
threadBlockPositions.clear();
threadBytePositions.clear();
blockState.clear();
threads = null;
Utility.writeToFile(metadata, DownloadMission.this);
}
private void initializer() {
init = runAsync(DownloadInitializer.mId, new DownloadInitializer(this));
}
/**
* Write this {@link DownloadMission} to the meta file asynchronously
* if no thread is already running.
*/
public void writeThisToFile() {
if (!mWritingToFile) {
mWritingToFile = true;
new Thread() {
@Override
public void run() {
doWriteThisToFile();
mWritingToFile = false;
}
}.start();
}
}
/**
* Write this {@link DownloadMission} to the meta file.
*/
private void doWriteThisToFile() {
private void writeThisToFile() {
synchronized (blockState) {
Utility.writeToFile(getMetaFilename(), this);
if (deleted) return;
Utility.writeToFile(metadata, DownloadMission.this);
}
mWritingToFile = false;
}
public boolean isFinished() {
return current >= urls.length && postprocessingName == null;
}
private boolean doPostprocessing() {
if (postprocessingName == null) return true;
try {
notifyPostProcessing(true);
notifyProgress(0);
Thread.currentThread().setName("[" + TAG + "] post-processing = " + postprocessingName + " filename = " + name);
Postprocessing algorithm = Postprocessing.getAlgorithm(postprocessingName, this);
algorithm.run();
} catch (Exception err) {
StringBuilder args = new StringBuilder(" ");
if (postprocessingArgs != null) {
for (String arg : postprocessingArgs) {
args.append(", ");
args.append(arg);
}
args.delete(0, 1);
}
Log.e(TAG, String.format("Post-processing failed. algorithm = %s args = [%s]", postprocessingName, args), err);
notifyError(ERROR_POSTPROCESSING_FAILED, err);
return false;
} finally {
notifyPostProcessing(false);
}
if (errCode != ERROR_NOTHING) notify(DownloadManagerService.MESSAGE_ERROR);
return errCode == ERROR_NOTHING;
}
private boolean deleteThisFromFile() {
synchronized (blockState) {
return metadata.delete();
}
}
private void readObject(ObjectInputStream inputStream)
throws java.io.IOException, ClassNotFoundException
{
inputStream.defaultReadObject();
mListeners = new ArrayList<>();
}
private void deleteThisFromFile() {
new File(getMetaFilename()).delete();
}
/**
* Get the path of the meta file
* run a method in a new thread
*
* @return the path to the meta file
* @param id id of new thread (used for debugging only)
* @param who the object whose {@code run} method is invoked when this thread is started
* @return the created thread
*/
private String getMetaFilename() {
return location + "/" + name + ".giga";
private Thread runAsync(int id, Runnable who) {
// known thread ids:
// -2: state saving by notifyProgress() method
// -1: wait for saving the state by pause() method
// 0: initializer
// >=1: any download thread
Thread thread = new Thread(who);
if (DEBUG) {
thread.setName(String.format("[%s] id = %s filename = %s", TAG, id, name));
}
thread.start();
return thread;
}
public File getDownloadedFile() {
return new File(location, name);
}
static class HttpError extends Exception {
int statusCode;
HttpError(int statusCode) {
this.statusCode = statusCode;
}
@Override
public String getMessage() {
return "Http status code" + String.valueOf(statusCode);
}
}
}

View file

@ -2,9 +2,12 @@ package us.shandian.giga.get;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.FileNotFoundException;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.ClosedByInterruptException;
import static org.schabi.newpipe.BuildConfig.DEBUG;
@ -18,7 +21,7 @@ public class DownloadRunnable implements Runnable {
private final DownloadMission mMission;
private final int mId;
public DownloadRunnable(DownloadMission mission, int id) {
DownloadRunnable(DownloadMission mission, int id) {
if (mission == null) throw new NullPointerException("mission is null");
mMission = mission;
mId = id;
@ -27,14 +30,25 @@ public class DownloadRunnable implements Runnable {
@Override
public void run() {
boolean retry = mMission.recovered;
long position = mMission.getPosition(mId);
long blockPosition = mMission.getBlockPosition(mId);
int retryCount = 0;
if (DEBUG) {
Log.d(TAG, mId + ":default pos " + position);
Log.d(TAG, mId + ":default pos " + blockPosition);
Log.d(TAG, mId + ":recovered: " + mMission.recovered);
}
while (mMission.errCode == -1 && mMission.running && position < mMission.blocks) {
BufferedInputStream ipt = null;
RandomAccessFile f;
try {
f = new RandomAccessFile(mMission.getDownloadedFile(), "rw");
} catch (FileNotFoundException e) {
mMission.notifyError(e);// this never should happen
return;
}
while (mMission.errCode == DownloadMission.ERROR_NOTHING && mMission.running && blockPosition < mMission.blocks) {
if (Thread.currentThread().isInterrupted()) {
mMission.pause();
@ -42,57 +56,47 @@ public class DownloadRunnable implements Runnable {
}
if (DEBUG && retry) {
Log.d(TAG, mId + ":retry is true. Resuming at " + position);
Log.d(TAG, mId + ":retry is true. Resuming at " + blockPosition);
}
// Wait for an unblocked position
while (!retry && position < mMission.blocks && mMission.isBlockPreserved(position)) {
while (!retry && blockPosition < mMission.blocks && mMission.isBlockPreserved(blockPosition)) {
if (DEBUG) {
Log.d(TAG, mId + ":position " + position + " preserved, passing");
Log.d(TAG, mId + ":position " + blockPosition + " preserved, passing");
}
position++;
blockPosition++;
}
retry = false;
if (position >= mMission.blocks) {
if (blockPosition >= mMission.blocks) {
break;
}
if (DEBUG) {
Log.d(TAG, mId + ":preserving position " + position);
Log.d(TAG, mId + ":preserving position " + blockPosition);
}
mMission.preserveBlock(position);
mMission.setPosition(mId, position);
mMission.preserveBlock(blockPosition);
mMission.setBlockPosition(mId, blockPosition);
long start = position * DownloadManager.BLOCK_SIZE;
long end = start + DownloadManager.BLOCK_SIZE - 1;
long start = (blockPosition * DownloadMission.BLOCK_SIZE) + mMission.getBlockBytePosition(mId);
long end = start + DownloadMission.BLOCK_SIZE - 1;
if (end >= mMission.length) {
end = mMission.length - 1;
}
HttpURLConnection conn = null;
int total = 0;
try {
URL url = new URL(mMission.url);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Range", "bytes=" + start + "-" + end);
HttpURLConnection conn = mMission.openConnection(mId, start, end);
if (DEBUG) {
Log.d(TAG, mId + ":" + conn.getRequestProperty("Range"));
Log.d(TAG, mId + ":Content-Length=" + conn.getContentLength() + " Code:" + conn.getResponseCode());
}
// A server may be ignoring the range request
// The server may be ignoring the range request
if (conn.getResponseCode() != 206) {
mMission.errCode = DownloadMission.ERROR_SERVER_UNSUPPORTED;
notifyError();
mMission.notifyError(new DownloadMission.HttpError(conn.getResponseCode()));
if (DEBUG) {
Log.e(TAG, mId + ":Unsupported " + conn.getResponseCode());
@ -101,76 +105,67 @@ public class DownloadRunnable implements Runnable {
break;
}
RandomAccessFile f = new RandomAccessFile(mMission.location + "/" + mMission.name, "rw");
f.seek(start);
java.io.InputStream ipt = conn.getInputStream();
byte[] buf = new byte[64*1024];
f.seek(mMission.offsets[mMission.current] + start);
while (start < end && mMission.running) {
int len = ipt.read(buf, 0, buf.length);
ipt = new BufferedInputStream(conn.getInputStream());
byte[] buf = new byte[DownloadMission.BUFFER_SIZE];
int len;
if (len == -1) {
break;
} else {
start += len;
total += len;
f.write(buf, 0, len);
notifyProgress(len);
}
while (start < end && mMission.running && (len = ipt.read(buf, 0, buf.length)) != -1) {
f.write(buf, 0, len);
start += len;
total += len;
mMission.notifyProgress(len);
}
if (DEBUG && mMission.running) {
Log.d(TAG, mId + ":position " + position + " finished, total length " + total);
Log.d(TAG, mId + ":position " + blockPosition + " finished, total length " + total);
}
f.close();
ipt.close();
// TODO We should save progress for each thread
// if the download is paused, save progress for this thread
if (!mMission.running) {
mMission.setThreadBytePosition(mId, total);
break;
}
} catch (Exception e) {
// TODO Retry count limit & notify error
retry = true;
mMission.setThreadBytePosition(mId, total);
notifyProgress(-total);
if (e instanceof ClosedByInterruptException) break;
if (retryCount++ > mMission.maxRetry) {
mMission.notifyError(e);
break;
}
if (DEBUG) {
Log.d(TAG, mId + ":position " + position + " retrying", e);
Log.d(TAG, mId + ":position " + blockPosition + " retrying due exception", e);
}
}
}
try {
f.close();
} catch (Exception err) {
// ¿ejected media storage? ¿file deleted? ¿storage ran out of space?
}
try {
if (ipt != null) ipt.close();
} catch (Exception err) {
// nothing to do
}
if (DEBUG) {
Log.d(TAG, "thread " + mId + " exited main loop");
Log.d(TAG, "thread " + mId + " exited from main download loop");
}
if (mMission.errCode == -1 && mMission.running) {
if (mMission.errCode == DownloadMission.ERROR_NOTHING && mMission.running) {
if (DEBUG) {
Log.d(TAG, "no error has happened, notifying");
}
notifyFinished();
mMission.notifyFinished();
}
if (DEBUG && !mMission.running) {
Log.d(TAG, "The mission has been paused. Passing.");
}
}
private void notifyProgress(final long len) {
synchronized (mMission) {
mMission.notifyProgress(len);
}
}
private void notifyError() {
synchronized (mMission) {
mMission.notifyError(DownloadMission.ERROR_SERVER_UNSUPPORTED);
mMission.pause();
}
}
private void notifyFinished() {
synchronized (mMission) {
mMission.notifyFinished();
}
}
}

View file

@ -1,74 +1,109 @@
package us.shandian.giga.get;
import android.support.annotation.NonNull;
import android.util.Log;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.ClosedByInterruptException;
import static org.schabi.newpipe.BuildConfig.DEBUG;
// Single-threaded fallback mode
public class DownloadRunnableFallback implements Runnable {
private final DownloadMission mMission;
//private int mId;
private static final String TAG = "DownloadRunnableFallbac";
public DownloadRunnableFallback(DownloadMission mission) {
if (mission == null) throw new NullPointerException("mission is null");
//mId = id;
private final DownloadMission mMission;
private int retryCount = 0;
private BufferedInputStream ipt;
private RandomAccessFile f;
DownloadRunnableFallback(@NonNull DownloadMission mission) {
mMission = mission;
ipt = null;
f = null;
}
private void dispose() {
try {
if (ipt != null) ipt.close();
} catch (IOException e) {
// nothing to do
}
try {
if (f != null) f.close();
} catch (IOException e) {
// ¿ejected media storage? ¿file deleted? ¿storage ran out of space?
}
}
@Override
public void run() {
try {
URL url = new URL(mMission.url);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
boolean done;
if (conn.getResponseCode() != 200 && conn.getResponseCode() != 206) {
notifyError(DownloadMission.ERROR_SERVER_UNSUPPORTED);
} else {
RandomAccessFile f = new RandomAccessFile(mMission.location + "/" + mMission.name, "rw");
f.seek(0);
BufferedInputStream ipt = new BufferedInputStream(conn.getInputStream());
byte[] buf = new byte[512];
int len = 0;
int start = 0;
while ((len = ipt.read(buf, 0, 512)) != -1 && mMission.running) {
f.write(buf, 0, len);
notifyProgress(len);
if (Thread.interrupted()) {
break;
}
}
f.close();
ipt.close();
if (!mMission.unknownLength) {
start = mMission.getBlockBytePosition(0);
if (DEBUG && start > 0) {
Log.i(TAG, "Resuming a single-thread download at " + start);
}
}
try {
int rangeStart = (mMission.unknownLength || start < 1) ? -1 : start;
HttpURLConnection conn = mMission.openConnection(1, rangeStart, -1);
// secondary check for the file length
if (!mMission.unknownLength) mMission.unknownLength = conn.getContentLength() == -1;
f = new RandomAccessFile(mMission.getDownloadedFile(), "rw");
f.seek(mMission.offsets[mMission.current] + start);
ipt = new BufferedInputStream(conn.getInputStream());
byte[] buf = new byte[DownloadMission.BUFFER_SIZE];
int len = 0;
while (mMission.running && (len = ipt.read(buf, 0, buf.length)) != -1) {
f.write(buf, 0, len);
start += len;
mMission.notifyProgress(len);
if (Thread.interrupted()) break;
}
// if thread goes interrupted check if the last part is written. This avoid re-download the whole file
done = len == -1;
} catch (Exception e) {
notifyError(DownloadMission.ERROR_UNKNOWN);
dispose();
// save position
mMission.setThreadBytePosition(0, start);
if (e instanceof ClosedByInterruptException) return;
if (retryCount++ > mMission.maxRetry) {
mMission.notifyError(e);
return;
}
run();// try again
return;
}
if (mMission.errCode == -1 && mMission.running) {
notifyFinished();
}
}
dispose();
private void notifyProgress(final long len) {
synchronized (mMission) {
mMission.notifyProgress(len);
}
}
private void notifyError(final int err) {
synchronized (mMission) {
mMission.notifyError(err);
mMission.pause();
}
}
private void notifyFinished() {
synchronized (mMission) {
if (done) {
mMission.notifyFinished();
} else {
mMission.setThreadBytePosition(0, start);
}
}
}

View file

@ -0,0 +1,16 @@
package us.shandian.giga.get;
public class FinishedMission extends Mission {
public FinishedMission() {
}
public FinishedMission(DownloadMission mission) {
source = mission.source;
length = mission.length;// ¿or mission.done?
timestamp = mission.timestamp;
name = mission.name;
location = mission.location;
kind = mission.kind;
}
}

View file

@ -0,0 +1,66 @@
package us.shandian.giga.get;
import java.io.File;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.Calendar;
public abstract class Mission implements Serializable {
private static final long serialVersionUID = 0L;// last bump: 5 october 2018
/**
* Source url of the resource
*/
public String source;
/**
* Length of the current resource
*/
public long length;
/**
* creation timestamp (and maybe unique identifier)
*/
public long timestamp;
/**
* The filename
*/
public String name;
/**
* The directory to store the download
*/
public String location;
/**
* pre-defined content type
*/
public char kind;
/**
* get the target file on the storage
*
* @return File object
*/
public File getDownloadedFile() {
return new File(location, name);
}
public boolean delete() {
deleted = true;
return getDownloadedFile().delete();
}
/**
* Indicate if this mission is deleted whatever is stored
*/
public transient boolean deleted = false;
@Override
public String toString() {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(timestamp);
return "[" + calendar.getTime().toString() + "] " + location + File.separator + name;
}
}

View file

@ -0,0 +1,73 @@
package us.shandian.giga.get.sqlite;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.util.Log;
import java.util.ArrayList;
import java.util.List;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.get.FinishedMission;
import us.shandian.giga.get.Mission;
import static us.shandian.giga.get.sqlite.DownloadMissionHelper.KEY_LOCATION;
import static us.shandian.giga.get.sqlite.DownloadMissionHelper.KEY_NAME;
import static us.shandian.giga.get.sqlite.DownloadMissionHelper.MISSIONS_TABLE_NAME;
public class DownloadDataSource {
private static final String TAG = "DownloadDataSource";
private final DownloadMissionHelper downloadMissionHelper;
public DownloadDataSource(Context context) {
downloadMissionHelper = new DownloadMissionHelper(context);
}
public ArrayList<FinishedMission> loadFinishedMissions() {
SQLiteDatabase database = downloadMissionHelper.getReadableDatabase();
Cursor cursor = database.query(MISSIONS_TABLE_NAME, null, null,
null, null, null, DownloadMissionHelper.KEY_TIMESTAMP);
int count = cursor.getCount();
if (count == 0) return new ArrayList<>(1);
ArrayList<FinishedMission> result = new ArrayList<>(count);
while (cursor.moveToNext()) {
result.add(DownloadMissionHelper.getMissionFromCursor(cursor));
}
return result;
}
public void addMission(DownloadMission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionHelper.getWritableDatabase();
ContentValues values = DownloadMissionHelper.getValuesOfMission(downloadMission);
database.insert(MISSIONS_TABLE_NAME, null, values);
}
public void deleteMission(Mission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionHelper.getWritableDatabase();
database.delete(MISSIONS_TABLE_NAME,
KEY_LOCATION + " = ? AND " +
KEY_NAME + " = ?",
new String[]{downloadMission.location, downloadMission.name});
}
public void updateMission(DownloadMission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionHelper.getWritableDatabase();
ContentValues values = DownloadMissionHelper.getValuesOfMission(downloadMission);
String whereClause = KEY_LOCATION + " = ? AND " +
KEY_NAME + " = ?";
int rowsAffected = database.update(MISSIONS_TABLE_NAME, values,
whereClause, new String[]{downloadMission.location, downloadMission.name});
if (rowsAffected != 1) {
Log.e(TAG, "Expected 1 row to be affected by update but got " + rowsAffected);
}
}
}

View file

@ -7,19 +7,19 @@ import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteOpenHelper;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.get.FinishedMission;
/**
* SqliteHelper to store {@link us.shandian.giga.get.DownloadMission}
* SQLiteHelper to store finished {@link us.shandian.giga.get.DownloadMission}'s
*/
public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper {
public class DownloadMissionHelper extends SQLiteOpenHelper {
private final String TAG = "DownloadMissionHelper";
// TODO: use NewPipeSQLiteHelper ('s constants) when playlist branch is merged (?)
private static final String DATABASE_NAME = "downloads.db";
private static final int DATABASE_VERSION = 2;
private static final int DATABASE_VERSION = 3;
/**
* The table name of download missions
*/
@ -30,9 +30,9 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper {
*/
static final String KEY_LOCATION = "location";
/**
* The key to the url of a mission
* The key to the urls of a mission
*/
static final String KEY_URL = "url";
static final String KEY_SOURCE_URL = "url";
/**
* The key to the name of a mission
*/
@ -45,6 +45,8 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper {
static final String KEY_TIMESTAMP = "timestamp";
static final String KEY_KIND = "kind";
/**
* The statement to create the table
*/
@ -52,16 +54,28 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper {
"CREATE TABLE " + MISSIONS_TABLE_NAME + " (" +
KEY_LOCATION + " TEXT NOT NULL, " +
KEY_NAME + " TEXT NOT NULL, " +
KEY_URL + " TEXT NOT NULL, " +
KEY_SOURCE_URL + " TEXT NOT NULL, " +
KEY_DONE + " INTEGER NOT NULL, " +
KEY_TIMESTAMP + " INTEGER NOT NULL, " +
KEY_KIND + " TEXT NOT NULL, " +
" UNIQUE(" + KEY_LOCATION + ", " + KEY_NAME + "));";
DownloadMissionSQLiteHelper(Context context) {
public DownloadMissionHelper(Context context) {
super(context, DATABASE_NAME, null, DATABASE_VERSION);
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL(MISSIONS_CREATE_TABLE);
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion == 2) {
db.execSQL("ALTER TABLE " + MISSIONS_TABLE_NAME + " ADD COLUMN " + KEY_KIND + " TEXT;");
}
}
/**
* Returns all values of the download mission as ContentValues.
*
@ -70,34 +84,29 @@ public class DownloadMissionSQLiteHelper extends SQLiteOpenHelper {
*/
public static ContentValues getValuesOfMission(DownloadMission downloadMission) {
ContentValues values = new ContentValues();
values.put(KEY_URL, downloadMission.url);
values.put(KEY_SOURCE_URL, downloadMission.source);
values.put(KEY_LOCATION, downloadMission.location);
values.put(KEY_NAME, downloadMission.name);
values.put(KEY_DONE, downloadMission.done);
values.put(KEY_TIMESTAMP, downloadMission.timestamp);
values.put(KEY_KIND, String.valueOf(downloadMission.kind));
return values;
}
@Override
public void onCreate(SQLiteDatabase db) {
db.execSQL(MISSIONS_CREATE_TABLE);
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
// Currently nothing to do
}
public static DownloadMission getMissionFromCursor(Cursor cursor) {
public static FinishedMission getMissionFromCursor(Cursor cursor) {
if (cursor == null) throw new NullPointerException("cursor is null");
int pos;
String name = cursor.getString(cursor.getColumnIndexOrThrow(KEY_NAME));
String location = cursor.getString(cursor.getColumnIndexOrThrow(KEY_LOCATION));
String url = cursor.getString(cursor.getColumnIndexOrThrow(KEY_URL));
DownloadMission mission = new DownloadMission(name, url, location);
mission.done = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_DONE));
String kind = cursor.getString(cursor.getColumnIndex(KEY_KIND));
if (kind == null || kind.isEmpty()) kind = "?";
FinishedMission mission = new FinishedMission();
mission.name = cursor.getString(cursor.getColumnIndexOrThrow(KEY_NAME));
mission.location = cursor.getString(cursor.getColumnIndexOrThrow(KEY_LOCATION));
mission.source = cursor.getString(cursor.getColumnIndexOrThrow(KEY_SOURCE_URL));;
mission.length = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_DONE));
mission.timestamp = cursor.getLong(cursor.getColumnIndexOrThrow(KEY_TIMESTAMP));
mission.finished = true;
mission.kind = kind.charAt(0);
return mission;
}
}

View file

@ -1,79 +0,0 @@
package us.shandian.giga.get.sqlite;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.util.Log;
import java.util.ArrayList;
import java.util.List;
import us.shandian.giga.get.DownloadDataSource;
import us.shandian.giga.get.DownloadMission;
import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.KEY_LOCATION;
import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.KEY_NAME;
import static us.shandian.giga.get.sqlite.DownloadMissionSQLiteHelper.MISSIONS_TABLE_NAME;
/**
* Non-thread-safe implementation of {@link DownloadDataSource}
*/
public class SQLiteDownloadDataSource implements DownloadDataSource {
private static final String TAG = "DownloadDataSourceImpl";
private final DownloadMissionSQLiteHelper downloadMissionSQLiteHelper;
public SQLiteDownloadDataSource(Context context) {
downloadMissionSQLiteHelper = new DownloadMissionSQLiteHelper(context);
}
@Override
public List<DownloadMission> loadMissions() {
ArrayList<DownloadMission> result;
SQLiteDatabase database = downloadMissionSQLiteHelper.getReadableDatabase();
Cursor cursor = database.query(MISSIONS_TABLE_NAME, null, null,
null, null, null, DownloadMissionSQLiteHelper.KEY_TIMESTAMP);
int count = cursor.getCount();
if (count == 0) return new ArrayList<>();
result = new ArrayList<>(count);
while (cursor.moveToNext()) {
result.add(DownloadMissionSQLiteHelper.getMissionFromCursor(cursor));
}
return result;
}
@Override
public void addMission(DownloadMission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase();
ContentValues values = DownloadMissionSQLiteHelper.getValuesOfMission(downloadMission);
database.insert(MISSIONS_TABLE_NAME, null, values);
}
@Override
public void updateMission(DownloadMission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase();
ContentValues values = DownloadMissionSQLiteHelper.getValuesOfMission(downloadMission);
String whereClause = KEY_LOCATION + " = ? AND " +
KEY_NAME + " = ?";
int rowsAffected = database.update(MISSIONS_TABLE_NAME, values,
whereClause, new String[]{downloadMission.location, downloadMission.name});
if (rowsAffected != 1) {
Log.e(TAG, "Expected 1 row to be affected by update but got " + rowsAffected);
}
}
@Override
public void deleteMission(DownloadMission downloadMission) {
if (downloadMission == null) throw new NullPointerException("downloadMission is null");
SQLiteDatabase database = downloadMissionSQLiteHelper.getWritableDatabase();
database.delete(MISSIONS_TABLE_NAME,
KEY_LOCATION + " = ? AND " +
KEY_NAME + " = ?",
new String[]{downloadMission.location, downloadMission.name});
}
}