Sulkta-OSS/straw
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

misc changes

Browse source 
* OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported)
* WebMReader: use int for SimpleBlock.dataSize instead of long
* Download Recovery: allow recovering uninitialized downloads
* check range-requests using HEAD method instead of GET
* DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix
* reporting downloads errors now include the source url with the selected quality and format
This commit is contained in:
kapodamy 2019-09-30 23:52:49 -03:00
parent 570738190d
commit 4292ca94ff
11 changed files with 294 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

35
app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
View file

@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException;
import us.shandian.giga.util.Utility;
import static org.schabi.newpipe.BuildConfig.DEBUG;
import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN;
public class DownloadInitializer extends Thread {
private final static String TAG = "DownloadInitializer";
@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread {
mConn = null;
}
private static void safeClose(HttpURLConnection con) {
private void dispose() {
try {
con.getInputStream().close();
mConn.getInputStream().close();
} catch (Exception e) {
// nothing to do
}
@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread {
long lowestSize = Long.MAX_VALUE;
for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1);
mConn = mMission.openConnection(mMission.urls[i], true, -1, -1);
mMission.establishConnection(mId, mConn);
safeClose(mConn);
dispose();
if (Thread.interrupted()) return;
long length = Utility.getContentLength(mConn);
@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread {
}
} else {
// ask for the current resource length
mConn = mMission.openConnection(mId, -1, -1);
mConn = mMission.openConnection(true, -1, -1);
mMission.establishConnection(mId, mConn);
safeClose(mConn);
dispose();
if (!mMission.running || Thread.interrupted()) return;
@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread {
}
} else {
// Open again
mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length);
mConn = mMission.openConnection(true, mMission.length - 10, mMission.length);
mMission.establishConnection(mId, mConn);
safeClose(mConn);
dispose();
if (!mMission.running || Thread.interrupted()) return;
@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread {
} catch (InterruptedIOException | ClosedByInterruptException e) {
return;
} catch (Exception e) {
if (!mMission.running) return;
if (!mMission.running || super.isInterrupted()) return;
if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
// for youtube streams. The url has expired
interrupt();
mMission.doRecover(e);
return;
}
if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread {
@Override
public void interrupt() {
super.interrupt();
if (mConn != null) {
try {
mConn.disconnect();
} catch (Exception e) {
// nothing to do
}
}
if (mConn != null) dispose();
}
}

36
app/src/main/java/us/shandian/giga/get/DownloadMission.java
View file

@ -204,22 +204,24 @@ public class DownloadMission extends Mission {
/**
* Opens a connection
*
* @param threadId id of the calling thread, used only for debugging
* @param rangeStart range start
* @param rangeEnd range end
* @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used
* @param rangeStart range start
* @param rangeEnd range end
* @return a {@link java.net.URLConnection URLConnection} linking to the URL.
* @throws IOException if an I/O exception occurs.
*/
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException {
return openConnection(urls[current], threadId, rangeStart, rangeEnd);
HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
return openConnection(urls[current], headRequest, rangeStart, rangeEnd);
}
HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException {
HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT);
conn.setRequestProperty("Accept", "*/*");
if (headRequest) conn.setRequestMethod("HEAD");
// BUG workaround: switching between networks can freeze the download forever
conn.setConnectTimeout(30000);
conn.setReadTimeout(10000);
@ -229,10 +231,6 @@ public class DownloadMission extends Mission {
if (rangeEnd > 0) req += rangeEnd;
conn.setRequestProperty("Range", req);
if (DEBUG) {
Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range"));
}
}
return conn;
@ -245,13 +243,14 @@ public class DownloadMission extends Mission {
* @throws HttpError if the HTTP Status-Code is not satisfiable
*/
void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError {
conn.connect();
int statusCode = conn.getResponseCode();
if (DEBUG) {
Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range"));
Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode);
}
switch (statusCode) {
case 204:
case 205:
@ -676,6 +675,15 @@ public class DownloadMission extends Mission {
return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished();
}
/**
* Indicates if mission urls has expired and there an attempt to renovate them
*
* @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false}
*/
public boolean isRecovering() {
return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive();
}
private boolean doPostprocessing() {
if (psAlgorithm == null || psState == 2) return true;
@ -742,10 +750,8 @@ public class DownloadMission extends Mission {
}
}
// set the current download url to null in case if the recovery
// process is canceled. Next time start() method is called the
// recovery will be executed, saving time
urls[current] = null;
errCode = ERROR_NOTHING;
errObject = null;
if (recoveryInfo[current].attempts >= maxRetry) {
recoveryInfo[current].attempts = 0;

146
app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java
View file

@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.HttpURLConnection;
import java.nio.channels.ClosedByInterruptException;
import java.util.List;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE;
public class DownloadMissionRecover extends Thread {
@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread {
static final int mID = -3;
private final DownloadMission mMission;
private final MissionRecoveryInfo mRecovery;
private final Exception mFromError;
private final boolean notInitialized;
private HttpURLConnection mConn;
private MissionRecoveryInfo mRecovery;
private StreamExtractor mExtractor;
DownloadMissionRecover(DownloadMission mission, Exception originError) {
mMission = mission;
mFromError = originError;
mRecovery = mission.recoveryInfo[mission.current];
notInitialized = mission.blocks == null && mission.current == 0;
}
@Override
@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread {
return;
}
/*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
return;
}*/
try {
/*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
return;
}*/
StreamingService svr = NewPipe.getServiceByUrl(mMission.source);
if (svr == null) {
throw new RuntimeException("Unknown source service");
}
StreamExtractor extractor = svr.getStreamExtractor(mMission.source);
extractor.fetchPage();
mExtractor = svr.getStreamExtractor(mMission.source);
mExtractor.fetchPage();
} catch (InterruptedIOException | ClosedByInterruptException e) {
return;
} catch (Exception e) {
if (!mMission.running || super.isInterrupted()) return;
mMission.notifyError(e);
return;
}
// maybe the following check is redundant
if (!mMission.running || super.isInterrupted()) return;
if (!notInitialized) {
// set the current download url to null in case if the recovery
// process is canceled. Next time start() method is called the
// recovery will be executed, saving time
mMission.urls[mMission.current] = null;
mRecovery = mMission.recoveryInfo[mMission.current];
resolveStream();
return;
}
Log.w(TAG, "mission is not fully initialized, this will take a while");
try {
for (; mMission.current < mMission.urls.length; mMission.current++) {
mRecovery = mMission.recoveryInfo[mMission.current];
if (test()) continue;
if (!mMission.running) return;
resolveStream();
if (!mMission.running) return;
// before continue, check if the current stream was resolved
if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) {
break;
}
}
} finally {
mMission.current = 0;
}
mMission.writeThisToFile();
if (!mMission.running || super.isInterrupted()) return;
mMission.running = false;
mMission.start();
}
private void resolveStream() {
if (mExtractor.getErrorMessage() != null) {
mMission.notifyError(mFromError);
return;
}
try {
String url = null;
switch (mMission.kind) {
switch (mRecovery.kind) {
case 'a':
for (AudioStream audio : extractor.getAudioStreams()) {
for (AudioStream audio : mExtractor.getAudioStreams()) {
if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) {
url = audio.getUrl();
break;
@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread {
case 'v':
List<VideoStream> videoStreams;
if (mRecovery.desired2)
videoStreams = extractor.getVideoOnlyStreams();
videoStreams = mExtractor.getVideoOnlyStreams();
else
videoStreams = extractor.getVideoStreams();
videoStreams = mExtractor.getVideoStreams();
for (VideoStream video : videoStreams) {
if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) {
url = video.getUrl();
@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread {
}
break;
case 's':
for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) {
for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) {
String tag = subtitles.getLanguageTag();
if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) {
url = subtitles.getURL();
@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread {
////// Validate the http resource doing a range request
/////////////////////
try {
mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length);
mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length);
mConn.setRequestProperty("If-Range", mRecovery.validateCondition);
mMission.establishConnection(mID, mConn);
@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread {
if (!mMission.running || e instanceof ClosedByInterruptException) return;
throw e;
} finally {
this.interrupt();
disconnect();
}
}
private void recover(String url, boolean stale) {
Log.i(TAG,
String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url)
String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url)
);
mMission.urls[mMission.current] = url;
mRecovery.attempts = 0;
if (url == null) {
mMission.notifyError(ERROR_RESOURCE_GONE, null);
return;
}
mMission.urls[mMission.current] = url;
mRecovery.attempts = 0;
if (notInitialized) return;
if (stale) {
mMission.resetState(false, false, DownloadMission.ERROR_NOTHING);
@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread {
return range;
}
private boolean test() {
if (mMission.urls[mMission.current] == null) return false;
try {
mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1);
mMission.establishConnection(mID, mConn);
if (mConn.getResponseCode() == 200) return true;
} catch (Exception e) {
// nothing to do
} finally {
disconnect();
}
return false;
}
private void disconnect() {
try {
try {
mConn.getInputStream().close();
} finally {
mConn.disconnect();
}
} catch (Exception e) {
// nothing to do
} finally {
mConn = null;
}
}
@Override
public void interrupt() {
super.interrupt();
if (mConn != null) {
try {
mConn.disconnect();
} catch (Exception e) {
// nothing to do
}
}
if (mConn != null) disconnect();
}
}

2
app/src/main/java/us/shandian/giga/get/DownloadRunnable.java
View file

@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread {
}
try {
mConn = mMission.openConnection(mId, start, end);
mConn = mMission.openConnection(false, start, end);
mMission.establishConnection(mId, mConn);
// check if the download can be resumed

20
app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java
View file

@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread {
private void dispose() {
try {
if (mIs != null) mIs.close();
try {
if (mIs != null) mIs.close();
} finally {
mConn.disconnect();
}
} catch (IOException e) {
// nothing to do
}
@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread {
long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start;
int mId = 1;
mConn = mMission.openConnection(mId, rangeStart, -1);
mConn = mMission.openConnection(false, rangeStart, -1);
if (mRetryCount == 0 && rangeStart == -1) {
// workaround: bypass android connection pool
mConn.setRequestProperty("Range", "bytes=0-");
}
mMission.establishConnection(mId, mConn);
// check if the download can be resumed
@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread {
mMission.notifyProgress(len);
}
dispose();
// if thread goes interrupted check if the last part is written. This avoid re-download the whole file
done = len == -1;
} catch (Exception e) {
@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread {
if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
// for youtube streams. The url has expired, recover
mMission.doRecover(e);
dispose();
mMission.doRecover(e);
return;
}
@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread {
return;
}
dispose();
if (done) {
mMission.notifyFinished();
} else {

43
app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java
View file

@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
private static final long serialVersionUID = 0L;
//public static final String DIRECT_SOURCE = "direct-source://";
public MediaFormat format;
MediaFormat format;
String desired;
boolean desired2;
int desiredBitrate;
byte kind;
String validateCondition = null;
transient int attempts = 0;
String validateCondition = null;
public MissionRecoveryInfo(@NonNull Stream stream) {
if (stream instanceof AudioStream) {
desiredBitrate = ((AudioStream) stream).average_bitrate;
desired2 = false;
kind = 'a';
} else if (stream instanceof VideoStream) {
desired = ((VideoStream) stream).getResolution();
desired2 = ((VideoStream) stream).isVideoOnly();
kind = 'v';
} else if (stream instanceof SubtitlesStream) {
desired = ((SubtitlesStream) stream).getLanguageTag();
desired2 = ((SubtitlesStream) stream).isAutoGenerated();
kind = 's';
} else {
throw new RuntimeException("Unknown stream kind");
}
@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
if (format == null) throw new NullPointerException("Stream format cannot be null");
}
@NonNull
@Override
public String toString() {
String info;
StringBuilder str = new StringBuilder();
str.append("type=");
switch (kind) {
case 'a':
str.append("audio");
info = "bitrate=" + desiredBitrate;
break;
case 'v':
str.append("video");
info = "quality=" + desired + " videoOnly=" + desired2;
break;
case 's':
str.append("subtitles");
info = "language=" + desired + " autoGenerated=" + desired2;
break;
default:
info = "";
str.append("other");
}
str.append(" format=")
.append(format.getName())
.append(' ')
.append(info);
return str.toString();
}
@Override
public int describeContents() {
return 0;
@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
parcel.writeString(this.desired);
parcel.writeInt(this.desired2 ? 0x01 : 0x00);
parcel.writeInt(this.desiredBitrate);
parcel.writeByte(this.kind);
parcel.writeString(this.validateCondition);
}
@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
this.desired = parcel.readString();
this.desired2 = parcel.readInt() != 0x00;
this.desiredBitrate = parcel.readInt();
this.kind = parcel.readByte();
this.validateCondition = parcel.readString();
}

36
app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java
View file

@ -36,6 +36,7 @@ import android.widget.Toast;
import org.schabi.newpipe.BuildConfig;
import org.schabi.newpipe.R;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.report.ErrorActivity;
import org.schabi.newpipe.report.UserAction;
import org.schabi.newpipe.util.NavigationHelper;
@ -44,11 +45,11 @@ import java.io.File;
import java.lang.ref.WeakReference;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.get.FinishedMission;
import us.shandian.giga.get.Mission;
import us.shandian.giga.get.MissionRecoveryInfo;
import us.shandian.giga.io.StoredFileHelper;
import us.shandian.giga.service.DownloadManager;
import us.shandian.giga.service.DownloadManagerService;
@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
// hide on error
// show if current resource length is not fetched
// show if length is unknown
h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength));
h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength));
float progress;
if (mission.unknownLength) {
@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
break;
case ERROR_POSTPROCESSING:
case ERROR_POSTPROCESSING_HOLD:
showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
return;
case ERROR_INSUFFICIENT_STORAGE:
msg = R.string.error_insufficient_storage;
break;
case ERROR_UNKNOWN_EXCEPTION:
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error);
showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error);
return;
case ERROR_PROGRESS_LOST:
msg = R.string.error_progress_lost;
@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
} else if (mission.errObject == null) {
msgEx = "(not_decelerated_error_code)";
} else {
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg);
showError(mission, UserAction.DOWNLOAD_FAILED, msg);
return;
}
break;
@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) {
@StringRes final int mMsg = msg;
builder.setPositiveButton(R.string.error_report_title, (dialog, which) ->
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg)
showError(mission, UserAction.DOWNLOAD_FAILED, mMsg)
);
}
@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
.show();
}
private void showError(Exception exception, UserAction action, @StringRes int reason) {
private void showError(DownloadMission mission, UserAction action, @StringRes int reason) {
StringBuilder request = new StringBuilder(256);
request.append(mission.source);
request.append(" [");
if (mission.recoveryInfo != null) {
for (MissionRecoveryInfo recovery : mission.recoveryInfo)
request.append(" {").append(recovery.toString()).append("} ");
}
request.append("]");
String service;
try {
service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName();
} catch (Exception e) {
service = "-";
}
ErrorActivity.reportError(
mContext,
Collections.singletonList(exception),
mission.errObject,
null,
null,
ErrorActivity.ErrorInfo.make(action, "-", "-", reason)
ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason)
);
}