Space reserving tweaks for huge video resolutions

* improve space reserving, allows write better 4K/8K video data
* do not use cache dirs in the muxers, Android can force close NewPipe if the device is running out of storage. Is a aggressive cache cleaning >:/
* (for devs) webm & mkv are the same thing
* calculate the final file size inside of the mission, instead getting from the UI
* simplify ps algorithms constructors
* [missing old commit message] simplify the loading of pending downloads
This commit is contained in:
kapodamy 2019-04-25 00:34:29 -03:00
parent 34b2b96158
commit 7b948f83c3
11 changed files with 608 additions and 550 deletions

View file

@ -6,10 +6,10 @@ import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
public class M4aNoDash extends Postprocessing {
class M4aNoDash extends Postprocessing {
M4aNoDash() {
super(0, true);
super(false, true, ALGORITHM_M4A_NO_DASH);
}
@Override

View file

@ -11,7 +11,7 @@ import java.io.IOException;
class Mp4FromDashMuxer extends Postprocessing {
Mp4FromDashMuxer() {
super(3 * 1024 * 1024/* 3 MiB */, true);
super(true, true, ALGORITHM_MP4_FROM_DASH_MUXER);
}
@Override

View file

@ -1,247 +1,256 @@
package us.shandian.giga.postprocessing;
import android.os.Message;
import android.support.annotation.NonNull;
import android.util.Log;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.io.ChunkFileInputStream;
import us.shandian.giga.io.CircularFileWriter;
import us.shandian.giga.io.CircularFileWriter.OffsetChecker;
import us.shandian.giga.service.DownloadManagerService;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD;
import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION;
public abstract class Postprocessing implements Serializable {
static transient final byte OK_RESULT = ERROR_NOTHING;
public transient static final String ALGORITHM_TTML_CONVERTER = "ttml";
public transient static final String ALGORITHM_WEBM_MUXER = "webm";
public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args, @NonNull File cacheDir) {
Postprocessing instance;
switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER:
instance = new TtmlConverter();
break;
case ALGORITHM_WEBM_MUXER:
instance = new WebMMuxer();
break;
case ALGORITHM_MP4_FROM_DASH_MUXER:
instance = new Mp4FromDashMuxer();
break;
case ALGORITHM_M4A_NO_DASH:
instance = new M4aNoDash();
break;
/*case "example-algorithm":
instance = new ExampleAlgorithm();*/
default:
throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName);
}
instance.args = args;
instance.name = algorithmName;// for debug only, maybe remove this field in the future
instance.cacheDir = cacheDir;
return instance;
}
/**
* Get a boolean value that indicate if the given algorithm work on the same
* file
*/
public boolean worksOnSameFile;
/**
* Get the recommended space to reserve for the given algorithm. The amount
* is in bytes
*/
public int recommendedReserve;
/**
* the download to post-process
*/
protected transient DownloadMission mission;
public transient File cacheDir;
private String[] args;
private String name;
Postprocessing(int recommendedReserve, boolean worksOnSameFile) {
this.recommendedReserve = recommendedReserve;
this.worksOnSameFile = worksOnSameFile;
}
public void run(DownloadMission target) throws IOException {
this.mission = target;
File temp = null;
CircularFileWriter out = null;
int result;
long finalLength = -1;
mission.done = 0;
mission.length = mission.storage.length();
if (worksOnSameFile) {
ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
try {
int i = 0;
for (; i < sources.length - 1; i++) {
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]);
}
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]);
if (test(sources)) {
for (SharpStream source : sources) source.rewind();
OffsetChecker checker = () -> {
for (ChunkFileInputStream source : sources) {
/*
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
* or the CircularFileWriter can lead to unexpected results
*/
if (source.isClosed() || source.available() < 1) {
continue;// the selected source is not used anymore
}
return source.getFilePointer() - 1;
}
return -1;
};
temp = new File(cacheDir, mission.storage.getName() + ".tmp");
out = new CircularFileWriter(mission.storage.getStream(), temp, checker);
out.onProgress = this::progressReport;
out.onWriteError = (err) -> {
mission.psState = 3;
mission.notifyError(ERROR_POSTPROCESSING_HOLD, err);
try {
synchronized (this) {
while (mission.psState == 3)
wait();
}
} catch (InterruptedException e) {
// nothing to do
Log.e(this.getClass().getSimpleName(), "got InterruptedException");
}
return mission.errCode == ERROR_NOTHING;
};
result = process(out, sources);
if (result == OK_RESULT)
finalLength = out.finalizeFile();
} else {
result = OK_RESULT;
}
} finally {
for (SharpStream source : sources) {
if (source != null && !source.isClosed()) {
source.close();
}
}
if (out != null) {
out.close();
}
if (temp != null) {
//noinspection ResultOfMethodCallIgnored
temp.delete();
}
}
} else {
result = test() ? process(null) : OK_RESULT;
}
if (result == OK_RESULT) {
if (finalLength != -1) {
mission.done = finalLength;
mission.length = finalLength;
}
} else {
mission.errCode = ERROR_UNKNOWN_EXCEPTION;
mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
}
if (result != OK_RESULT && worksOnSameFile) mission.storage.delete();
this.mission = null;
}
/**
* Test if the post-processing algorithm can be skipped
*
* @param sources files to be processed
* @return {@code true} if the post-processing is required, otherwise, {@code false}
* @throws IOException if an I/O error occurs.
*/
boolean test(SharpStream... sources) throws IOException {
return true;
}
/**
* Abstract method to execute the post-processing algorithm
*
* @param out output stream
* @param sources files to be processed
* @return a error code, 0 means the operation was successful
* @throws IOException if an I/O error occurs.
*/
abstract int process(SharpStream out, SharpStream... sources) throws IOException;
String getArgumentAt(int index, String defaultValue) {
if (args == null || index >= args.length) {
return defaultValue;
}
return args[index];
}
private void progressReport(long done) {
mission.done = done;
if (mission.length < mission.done) mission.length = mission.done;
Message m = new Message();
m.what = DownloadManagerService.MESSAGE_PROGRESS;
m.obj = mission;
mission.mHandler.sendMessage(m);
}
@NonNull
@Override
public String toString() {
StringBuilder str = new StringBuilder();
str.append("name=").append(name).append('[');
if (args != null) {
for (String arg : args) {
str.append(", ");
str.append(arg);
}
str.delete(0, 1);
}
return str.append(']').toString();
}
}
package us.shandian.giga.postprocessing;
import android.os.Message;
import android.support.annotation.NonNull;
import android.util.Log;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.io.ChunkFileInputStream;
import us.shandian.giga.io.CircularFileWriter;
import us.shandian.giga.io.CircularFileWriter.OffsetChecker;
import us.shandian.giga.service.DownloadManagerService;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD;
import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION;
public abstract class Postprocessing implements Serializable {
static transient final byte OK_RESULT = ERROR_NOTHING;
public transient static final String ALGORITHM_TTML_CONVERTER = "ttml";
public transient static final String ALGORITHM_WEBM_MUXER = "webm";
public transient static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
public transient static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
public static Postprocessing getAlgorithm(@NonNull String algorithmName, String[] args) {
Postprocessing instance;
switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER:
instance = new TtmlConverter();
break;
case ALGORITHM_WEBM_MUXER:
instance = new WebMMuxer();
break;
case ALGORITHM_MP4_FROM_DASH_MUXER:
instance = new Mp4FromDashMuxer();
break;
case ALGORITHM_M4A_NO_DASH:
instance = new M4aNoDash();
break;
/*case "example-algorithm":
instance = new ExampleAlgorithm();*/
default:
throw new UnsupportedOperationException("Unimplemented post-processing algorithm: " + algorithmName);
}
instance.args = args;
return instance;
}
/**
* Get a boolean value that indicate if the given algorithm work on the same
* file
*/
public final boolean worksOnSameFile;
/**
* Indicates whether the selected algorithm needs space reserved at the beginning of the file
*/
public final boolean reserveSpace;
/**
* Gets the given algorithm short name
*/
private final String name;
private String[] args;
protected transient DownloadMission mission;
private File tempFile;
Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) {
this.reserveSpace = reserveSpace;
this.worksOnSameFile = worksOnSameFile;
this.name = algorithmName;// for debugging only
}
public void setTemporalDir(@NonNull File directory) {
long rnd = (int) (Math.random() * 100000f);
tempFile = new File(directory, rnd + "_" + System.nanoTime() + ".tmp");
}
public void cleanupTemporalDir() {
if (tempFile != null && tempFile.exists()) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
}
}
public void run(DownloadMission target) throws IOException {
this.mission = target;
CircularFileWriter out = null;
int result;
long finalLength = -1;
mission.done = 0;
mission.length = mission.storage.length();
if (worksOnSameFile) {
ChunkFileInputStream[] sources = new ChunkFileInputStream[mission.urls.length];
try {
int i = 0;
for (; i < sources.length - 1; i++) {
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i], mission.offsets[i + 1]);
}
sources[i] = new ChunkFileInputStream(mission.storage.getStream(), mission.offsets[i]);
if (test(sources)) {
for (SharpStream source : sources) source.rewind();
OffsetChecker checker = () -> {
for (ChunkFileInputStream source : sources) {
/*
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
* or the CircularFileWriter can lead to unexpected results
*/
if (source.isClosed() || source.available() < 1) {
continue;// the selected source is not used anymore
}
return source.getFilePointer() - 1;
}
return -1;
};
out = new CircularFileWriter(mission.storage.getStream(), tempFile, checker);
out.onProgress = this::progressReport;
out.onWriteError = (err) -> {
mission.psState = 3;
mission.notifyError(ERROR_POSTPROCESSING_HOLD, err);
try {
synchronized (this) {
while (mission.psState == 3)
wait();
}
} catch (InterruptedException e) {
// nothing to do
Log.e(this.getClass().getSimpleName(), "got InterruptedException");
}
return mission.errCode == ERROR_NOTHING;
};
result = process(out, sources);
if (result == OK_RESULT)
finalLength = out.finalizeFile();
} else {
result = OK_RESULT;
}
} finally {
for (SharpStream source : sources) {
if (source != null && !source.isClosed()) {
source.close();
}
}
if (out != null) {
out.close();
}
if (tempFile != null) {
//noinspection ResultOfMethodCallIgnored
tempFile.delete();
tempFile = null;
}
}
} else {
result = test() ? process(null) : OK_RESULT;
}
if (result == OK_RESULT) {
if (finalLength != -1) {
mission.done = finalLength;
mission.length = finalLength;
}
} else {
mission.errCode = ERROR_UNKNOWN_EXCEPTION;
mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
}
if (result != OK_RESULT && worksOnSameFile) mission.storage.delete();
this.mission = null;
}
/**
* Test if the post-processing algorithm can be skipped
*
* @param sources files to be processed
* @return {@code true} if the post-processing is required, otherwise, {@code false}
* @throws IOException if an I/O error occurs.
*/
boolean test(SharpStream... sources) throws IOException {
return true;
}
/**
* Abstract method to execute the post-processing algorithm
*
* @param out output stream
* @param sources files to be processed
* @return a error code, 0 means the operation was successful
* @throws IOException if an I/O error occurs.
*/
abstract int process(SharpStream out, SharpStream... sources) throws IOException;
String getArgumentAt(int index, String defaultValue) {
if (args == null || index >= args.length) {
return defaultValue;
}
return args[index];
}
private void progressReport(long done) {
mission.done = done;
if (mission.length < mission.done) mission.length = mission.done;
Message m = new Message();
m.what = DownloadManagerService.MESSAGE_PROGRESS;
m.obj = mission;
mission.mHandler.sendMessage(m);
}
@NonNull
@Override
public String toString() {
StringBuilder str = new StringBuilder();
str.append("name=").append(name).append('[');
if (args != null) {
for (String arg : args) {
str.append(", ");
str.append(arg);
}
str.delete(0, 1);
}
return str.append(']').toString();
}
}

View file

@ -1,72 +1,72 @@
package us.shandian.giga.postprocessing;
import android.util.Log;
import org.schabi.newpipe.streams.SubtitleConverter;
import org.schabi.newpipe.streams.io.SharpStream;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.text.ParseException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
/**
* @author kapodamy
*/
class TtmlConverter extends Postprocessing {
private static final String TAG = "TtmlConverter";
TtmlConverter() {
// due how XmlPullParser works, the xml is fully loaded on the ram
super(0, true);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
// check if the subtitle is already in srt and copy, this should never happen
String format = getArgumentAt(0, null);
if (format == null || format.equals("ttml")) {
SubtitleConverter ttmlDumper = new SubtitleConverter();
try {
ttmlDumper.dumpTTML(
sources[0],
out,
getArgumentAt(1, "true").equals("true"),
getArgumentAt(2, "true").equals("true")
);
} catch (Exception err) {
Log.e(TAG, "subtitle parse failed", err);
if (err instanceof IOException) {
return 1;
} else if (err instanceof ParseException) {
return 2;
} else if (err instanceof SAXException) {
return 3;
} else if (err instanceof ParserConfigurationException) {
return 4;
} else if (err instanceof XPathExpressionException) {
return 7;
}
return 8;
}
return OK_RESULT;
} else if (format.equals("srt")) {
byte[] buffer = new byte[8 * 1024];
int read;
while ((read = sources[0].read(buffer)) > 0) {
out.write(buffer, 0, read);
}
return OK_RESULT;
}
throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format);
}
}
package us.shandian.giga.postprocessing;
import android.util.Log;
import org.schabi.newpipe.streams.SubtitleConverter;
import org.schabi.newpipe.streams.io.SharpStream;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.text.ParseException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
/**
* @author kapodamy
*/
class TtmlConverter extends Postprocessing {
private static final String TAG = "TtmlConverter";
TtmlConverter() {
// due how XmlPullParser works, the xml is fully loaded on the ram
super(false, true, ALGORITHM_TTML_CONVERTER);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
// check if the subtitle is already in srt and copy, this should never happen
String format = getArgumentAt(0, null);
if (format == null || format.equals("ttml")) {
SubtitleConverter ttmlDumper = new SubtitleConverter();
try {
ttmlDumper.dumpTTML(
sources[0],
out,
getArgumentAt(1, "true").equals("true"),
getArgumentAt(2, "true").equals("true")
);
} catch (Exception err) {
Log.e(TAG, "subtitle parse failed", err);
if (err instanceof IOException) {
return 1;
} else if (err instanceof ParseException) {
return 2;
} else if (err instanceof SAXException) {
return 3;
} else if (err instanceof ParserConfigurationException) {
return 4;
} else if (err instanceof XPathExpressionException) {
return 7;
}
return 8;
}
return OK_RESULT;
} else if (format.equals("srt")) {
byte[] buffer = new byte[8 * 1024];
int read;
while ((read = sources[0].read(buffer)) > 0) {
out.write(buffer, 0, read);
}
return OK_RESULT;
}
throw new UnsupportedOperationException("Can't convert this subtitle, unimplemented format: " + format);
}
}

View file

@ -1,44 +1,44 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.WebMReader.TrackKind;
import org.schabi.newpipe.streams.WebMReader.WebMTrack;
import org.schabi.newpipe.streams.WebMWriter;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
/**
* @author kapodamy
*/
class WebMMuxer extends Postprocessing {
WebMMuxer() {
super(5 * 1024 * 1024/* 5 MiB */, true);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
WebMWriter muxer = new WebMWriter(sources);
muxer.parseSources();
// youtube uses a webm with a fake video track that acts as a "cover image"
int[] indexes = new int[sources.length];
for (int i = 0; i < sources.length; i++) {
WebMTrack[] tracks = muxer.getTracksFromSource(i);
for (int j = 0; j < tracks.length; j++) {
if (tracks[j].kind == TrackKind.Audio) {
indexes[i] = j;
i = sources.length;
break;
}
}
}
muxer.selectTracks(indexes);
muxer.build(out);
return OK_RESULT;
}
}
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.WebMReader.TrackKind;
import org.schabi.newpipe.streams.WebMReader.WebMTrack;
import org.schabi.newpipe.streams.WebMWriter;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
/**
* @author kapodamy
*/
class WebMMuxer extends Postprocessing {
WebMMuxer() {
super(true, true, ALGORITHM_WEBM_MUXER);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
WebMWriter muxer = new WebMWriter(sources);
muxer.parseSources();
// youtube uses a webm with a fake video track that acts as a "cover image"
int[] indexes = new int[sources.length];
for (int i = 0; i < sources.length; i++) {
WebMTrack[] tracks = muxer.getTracksFromSource(i);
for (int j = 0; j < tracks.length; j++) {
if (tracks[j].kind == TrackKind.Audio) {
indexes[i] = j;
i = sources.length;
break;
}
}
}
muxer.selectTracks(indexes);
muxer.build(out);
return OK_RESULT;
}
}