New MP4 muxer + Queue changes + Storage fixes

Main changes:
* correctly check the available space (CircularFile.java)
* misc cleanup (CircularFile.java)
* use the "Error Reporter" for non-http errors
* rewrite network state checking and add better support for API 21 (Lollipop) or higher
* implement "metered networks"
* add buttons in "Downloads" activity to start/pause all pending downloads, ignoring the queue flag or if the network is "metered"
* add workaround for VPN connections and/or network switching. Example: switching WiFi to 3G
* rewrite DataReader ¡Webm muxer is now 57% more faster!
* rewrite CircularFile, use file buffers instead of memory buffers. Less troubles in low-end devices
* fix missing offset for KaxCluster (WebMWriter.java), manifested as no thumbnails on file explorers

Download queue:
* remember queue status, unless the user pause the download (un-queue)
* semi-automatic downloads, between networks. Effective if the user create a new download or the downloads activity is starts
* allow enqueue failed downloads
* new option, queue limit, enabled by default. Used to allow one or multiple downloads at same time

Miscellaneous:
* fix crash while selecting details/error menu (mistake on MissionFragment.java)
* misc serialize changes (DownloadMission.java)
* minor UI tweaks
* allow overwrite paused downloads
* fix wrong icons for grid/list button in downloads
* add share option
* implement #2006
* correct misspelled word in strings.xml (es) (cmn)
* fix MissionAdapter crash during device shutdown

New Mp4Muxer + required changes:
* new mp4 muxer (from dash only) with this, muxing on Android 7 is possible now!!!
* re-work in SharpStream
* drop mp4 dash muxer
* misc changes: add warning in SecondaryStreamHelper.java,
* strip m4a DASH files to normal m4a format (youtube only)

Fix storage issues:
* warn to the user if is choosing a "read only" download directory (for external SD Cards), useless is rooted :)
* "write proof" allow post-processing resuming only if the device ran out of space
* implement "insufficient storage" error for downloads
This commit is contained in:
kapodamy 2019-03-22 22:54:07 -03:00
parent 1684a2110c
commit 9e34fee58c
49 changed files with 2715 additions and 1936 deletions

View file

@ -0,0 +1,43 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.Mp4DashReader;
import org.schabi.newpipe.streams.Mp4FromDashWriter;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
public class M4aNoDash extends Postprocessing {
M4aNoDash(DownloadMission mission) {
super(mission, 0, true);
}
@Override
boolean test(SharpStream... sources) throws IOException {
// check if the mp4 file is DASH (youtube)
Mp4DashReader reader = new Mp4DashReader(sources[0]);
reader.parse();
switch (reader.getBrands()[0]) {
case 0x64617368:// DASH
case 0x69736F35:// ISO5
return true;
default:
return false;
}
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
Mp4FromDashWriter muxer = new Mp4FromDashWriter(sources[0]);
muxer.setMainBrand(0x4D344120);// binary string "M4A "
muxer.parseSources();
muxer.selectTracks(0);
muxer.build(out);
return OK_RESULT;
}
}

View file

@ -1,29 +1,29 @@
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.Mp4DashWriter;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
/**
* @author kapodamy
*/
class Mp4DashMuxer extends Postprocessing {
Mp4DashMuxer(DownloadMission mission) {
super(mission, 15360 * 1024/* 15 MiB */, true);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
Mp4DashWriter muxer = new Mp4DashWriter(sources);
muxer.parseSources();
muxer.selectTracks(0, 0);
muxer.build(out);
return OK_RESULT;
}
}
package us.shandian.giga.postprocessing;
import org.schabi.newpipe.streams.Mp4FromDashWriter;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
/**
* @author kapodamy
*/
class Mp4FromDashMuxer extends Postprocessing {
Mp4FromDashMuxer(DownloadMission mission) {
super(mission, 2 * 1024 * 1024/* 2 MiB */, true);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
Mp4FromDashWriter muxer = new Mp4FromDashWriter(sources);
muxer.parseSources();
muxer.selectTracks(0, 0);
muxer.build(out);
return OK_RESULT;
}
}

View file

@ -1,136 +0,0 @@
package us.shandian.giga.postprocessing;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaExtractor;
import android.media.MediaMuxer;
import android.media.MediaMuxer.OutputFormat;
import android.util.Log;
import static org.schabi.newpipe.BuildConfig.DEBUG;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import us.shandian.giga.get.DownloadMission;
class Mp4Muxer extends Postprocessing {
private static final String TAG = "Mp4Muxer";
private static final int NOTIFY_BYTES_INTERVAL = 128 * 1024;// 128 KiB
Mp4Muxer(DownloadMission mission) {
super(mission, 0, false);
}
@Override
int process(SharpStream out, SharpStream... sources) throws IOException {
File dlFile = mission.getDownloadedFile();
File tmpFile = new File(mission.location, mission.name.concat(".tmp"));
if (tmpFile.exists())
if (!tmpFile.delete()) return DownloadMission.ERROR_FILE_CREATION;
if (!tmpFile.createNewFile()) return DownloadMission.ERROR_FILE_CREATION;
FileInputStream source = null;
MediaMuxer muxer = null;
//noinspection TryFinallyCanBeTryWithResources
try {
source = new FileInputStream(dlFile);
MediaExtractor tracks[] = {
getMediaExtractor(source, mission.offsets[0], mission.offsets[1] - mission.offsets[0]),
getMediaExtractor(source, mission.offsets[1], mission.length - mission.offsets[1])
};
muxer = new MediaMuxer(tmpFile.getAbsolutePath(), OutputFormat.MUXER_OUTPUT_MPEG_4);
int tracksIndex[] = {
muxer.addTrack(tracks[0].getTrackFormat(0)),
muxer.addTrack(tracks[1].getTrackFormat(0))
};
ByteBuffer buffer = ByteBuffer.allocate(512 * 1024);// 512 KiB
BufferInfo info = new BufferInfo();
long written = 0;
long nextReport = NOTIFY_BYTES_INTERVAL;
muxer.start();
while (true) {
int done = 0;
for (int i = 0; i < tracks.length; i++) {
if (tracksIndex[i] < 0) continue;
info.set(0,
tracks[i].readSampleData(buffer, 0),
tracks[i].getSampleTime(),
tracks[i].getSampleFlags()
);
if (info.size >= 0) {
muxer.writeSampleData(tracksIndex[i], buffer, info);
written += info.size;
done++;
}
if (!tracks[i].advance()) {
// EOF reached
tracks[i].release();
tracksIndex[i] = -1;
}
if (written > nextReport) {
nextReport = written + NOTIFY_BYTES_INTERVAL;
super.progressReport(written);
}
}
if (done < 1) break;
}
// this part should not fail
if (!dlFile.delete()) return DownloadMission.ERROR_FILE_CREATION;
if (!tmpFile.renameTo(dlFile)) return DownloadMission.ERROR_FILE_CREATION;
return OK_RESULT;
} finally {
try {
if (muxer != null) {
muxer.stop();
muxer.release();
}
} catch (Exception err) {
if (DEBUG)
Log.e(TAG, "muxer stop/release failed", err);
}
if (source != null) {
try {
source.close();
} catch (IOException e) {
// nothing to do
}
}
// if the operation fails, delete the temporal file
if (tmpFile.exists()) {
//noinspection ResultOfMethodCallIgnored
tmpFile.delete();
}
}
}
private MediaExtractor getMediaExtractor(FileInputStream source, long offset, long length) throws IOException {
MediaExtractor extractor = new MediaExtractor();
extractor.setDataSource(source.getFD(), offset, length);
extractor.selectTrack(0);
return extractor;
}
}

View file

@ -1,6 +1,7 @@
package us.shandian.giga.postprocessing;
import android.os.Message;
import android.util.Log;
import org.schabi.newpipe.streams.io.SharpStream;
@ -9,17 +10,22 @@ import java.io.IOException;
import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.postprocessing.io.ChunkFileInputStream;
import us.shandian.giga.postprocessing.io.CircularFile;
import us.shandian.giga.postprocessing.io.CircularFileWriter;
import us.shandian.giga.postprocessing.io.CircularFileWriter.OffsetChecker;
import us.shandian.giga.service.DownloadManagerService;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_POSTPROCESSING_HOLD;
import static us.shandian.giga.get.DownloadMission.ERROR_UNKNOWN_EXCEPTION;
public abstract class Postprocessing {
static final byte OK_RESULT = DownloadMission.ERROR_NOTHING;
static final byte OK_RESULT = ERROR_NOTHING;
public static final String ALGORITHM_TTML_CONVERTER = "ttml";
public static final String ALGORITHM_MP4_DASH_MUXER = "mp4D";
public static final String ALGORITHM_MP4_MUXER = "mp4";
public static final String ALGORITHM_WEBM_MUXER = "webm";
public static final String ALGORITHM_MP4_FROM_DASH_MUXER = "mp4D-mp4";
public static final String ALGORITHM_M4A_NO_DASH = "mp4D-m4a";
public static Postprocessing getAlgorithm(String algorithmName, DownloadMission mission) {
if (null == algorithmName) {
@ -27,14 +33,14 @@ public abstract class Postprocessing {
} else switch (algorithmName) {
case ALGORITHM_TTML_CONVERTER:
return new TtmlConverter(mission);
case ALGORITHM_MP4_DASH_MUXER:
return new Mp4DashMuxer(mission);
case ALGORITHM_MP4_MUXER:
return new Mp4Muxer(mission);
case ALGORITHM_WEBM_MUXER:
return new WebMMuxer(mission);
case ALGORITHM_MP4_FROM_DASH_MUXER:
return new Mp4FromDashMuxer(mission);
case ALGORITHM_M4A_NO_DASH:
return new M4aNoDash(mission);
/*case "example-algorithm":
return new ExampleAlgorithm(mission);*/
return new ExampleAlgorithm(mission);*/
default:
throw new RuntimeException("Unimplemented post-processing algorithm: " + algorithmName);
}
@ -65,7 +71,8 @@ public abstract class Postprocessing {
public void run() throws IOException {
File file = mission.getDownloadedFile();
CircularFile out = null;
File temp = null;
CircularFileWriter out = null;
int result;
long finalLength = -1;
@ -81,29 +88,54 @@ public abstract class Postprocessing {
}
sources[i] = new ChunkFileInputStream(file, mission.offsets[i], mission.getDownloadedFile().length(), "rw");
int[] idx = {0};
CircularFile.OffsetChecker checker = () -> {
while (idx[0] < sources.length) {
/*
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
* or the CircularFile can lead to unexpected results
*/
if (sources[idx[0]].isDisposed() || sources[idx[0]].available() < 1) {
idx[0]++;
continue;// the selected source is not used anymore
if (test(sources)) {
for (SharpStream source : sources) source.rewind();
OffsetChecker checker = () -> {
for (ChunkFileInputStream source : sources) {
/*
* WARNING: never use rewind() in any chunk after any writing (especially on first chunks)
* or the CircularFileWriter can lead to unexpected results
*/
if (source.isDisposed() || source.available() < 1) {
continue;// the selected source is not used anymore
}
return source.getFilePointer() - 1;
}
return sources[idx[0]].getFilePointer() - 1;
}
return -1;
};
return -1;
};
out = new CircularFile(file, 0, this::progressReport, checker);
temp = new File(mission.location, mission.name + ".tmp");
result = process(out, sources);
out = new CircularFileWriter(file, temp, checker);
out.onProgress = this::progressReport;
if (result == OK_RESULT)
finalLength = out.finalizeFile();
out.onWriteError = (err) -> {
mission.postprocessingState = 3;
mission.notifyError(ERROR_POSTPROCESSING_HOLD, err);
try {
synchronized (this) {
while (mission.postprocessingState == 3)
wait();
}
} catch (InterruptedException e) {
// nothing to do
Log.e(this.getClass().getSimpleName(), "got InterruptedException");
}
return mission.errCode == ERROR_NOTHING;
};
result = process(out, sources);
if (result == OK_RESULT)
finalLength = out.finalizeFile();
} else {
result = OK_RESULT;
}
} finally {
for (SharpStream source : sources) {
if (source != null && !source.isDisposed()) {
@ -113,17 +145,22 @@ public abstract class Postprocessing {
if (out != null) {
out.dispose();
}
if (temp != null) {
//noinspection ResultOfMethodCallIgnored
temp.delete();
}
}
} else {
result = process(null);
result = test() ? process(null) : OK_RESULT;
}
if (result == OK_RESULT) {
if (finalLength < 0) finalLength = file.length();
mission.done = finalLength;
mission.length = finalLength;
if (finalLength != -1) {
mission.done = finalLength;
mission.length = finalLength;
}
} else {
mission.errCode = DownloadMission.ERROR_UNKNOWN_EXCEPTION;
mission.errCode = ERROR_UNKNOWN_EXCEPTION;
mission.errObject = new RuntimeException("post-processing algorithm returned " + result);
}
@ -134,7 +171,18 @@ public abstract class Postprocessing {
}
/**
* Abstract method to execute the pos-processing algorithm
* Test if the post-processing algorithm can be skipped
*
* @param sources files to be processed
* @return {@code true} if the post-processing is required, otherwise, {@code false}
* @throws IOException if an I/O error occurs.
*/
boolean test(SharpStream... sources) throws IOException {
return true;
}
/**
* Abstract method to execute the post-processing algorithm
*
* @param out output stream
* @param sources files to be processed
@ -151,7 +199,7 @@ public abstract class Postprocessing {
return mission.postprocessingArgs[index];
}
void progressReport(long done) {
private void progressReport(long done) {
mission.done = done;
if (mission.length < mission.done) mission.length = mission.done;

View file

@ -94,7 +94,7 @@ public class ChunkFileInputStream extends SharpStream {
}
@Override
public int available() {
public long available() {
return (int) (length - position);
}
@ -147,7 +147,4 @@ public class ChunkFileInputStream extends SharpStream {
public void write(byte[] buffer, int offset, int count) {
}
@Override
public void flush() {
}
}

View file

@ -1,375 +0,0 @@
package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
public class CircularFile extends SharpStream {
private final static int AUX_BUFFER_SIZE = 1024 * 1024;// 1 MiB
private final static int AUX_BUFFER_SIZE2 = 512 * 1024;// 512 KiB
private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB
private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB
private final static boolean IMMEDIATE_AUX_BUFFER_FLUSH = false;
private RandomAccessFile out;
private long position;
private long maxLengthKnown = -1;
private ArrayList<ManagedBuffer> auxiliaryBuffers;
private OffsetChecker callback;
private ManagedBuffer queue;
private long startOffset;
private ProgressReport onProgress;
private long reportPosition;
public CircularFile(File file, long offset, ProgressReport progressReport, OffsetChecker checker) throws IOException {
if (checker == null) {
throw new NullPointerException("checker is null");
}
try {
queue = new ManagedBuffer(QUEUE_BUFFER_SIZE);
out = new RandomAccessFile(file, "rw");
out.seek(offset);
position = offset;
} catch (IOException err) {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
// nothing to do
}
throw err;
}
auxiliaryBuffers = new ArrayList<>(15);
callback = checker;
startOffset = offset;
reportPosition = offset;
onProgress = progressReport;
}
/**
* Close the file without flushing any buffer
*/
@Override
public void dispose() {
try {
auxiliaryBuffers = null;
if (out != null) {
out.close();
out = null;
}
} catch (IOException err) {
// nothing to do
}
}
/**
* Flush any buffer and close the output file. Use this method if the
* operation is successful
*
* @return the final length of the file
* @throws IOException if an I/O error occurs
*/
public long finalizeFile() throws IOException {
flushEverything();
if (maxLengthKnown > -1) {
position = maxLengthKnown;
}
if (position < out.length()) {
out.setLength(position);
}
dispose();
return position;
}
@Override
public void write(byte b) throws IOException {
write(new byte[]{b}, 0, 1);
}
@Override
public void write(byte b[]) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte b[], int off, int len) throws IOException {
if (len == 0) {
return;
}
long end = callback.check();
long available;
if (end == -1) {
available = Long.MAX_VALUE;
} else {
if (end < startOffset) {
throw new IOException("The reported offset is invalid. reported offset is " + String.valueOf(end));
}
available = end - position;
}
// Check if possible flush one or more auxiliary buffer
if (auxiliaryBuffers.size() > 0) {
ManagedBuffer aux = auxiliaryBuffers.get(0);
// check if there is enough space to flush it completely
while (available >= (aux.size + queue.size)) {
available -= aux.size;
writeQueue(aux.buffer, 0, aux.size);
aux.dereference();
auxiliaryBuffers.remove(0);
if (auxiliaryBuffers.size() < 1) {
aux = null;
break;
}
aux = auxiliaryBuffers.get(0);
}
if (IMMEDIATE_AUX_BUFFER_FLUSH) {
// try partial flush to avoid allocate another auxiliary buffer
if (aux != null && aux.available() < len && available > queue.size) {
int size = Math.min(aux.size, (int) available - queue.size);
writeQueue(aux.buffer, 0, size);
aux.dereference(size);
available -= size;
}
}
}
if (auxiliaryBuffers.size() < 1 && available > (len + queue.size)) {
writeQueue(b, off, len);
} else {
int i = auxiliaryBuffers.size() - 1;
while (len > 0) {
if (i < 0) {
// allocate a new auxiliary buffer
auxiliaryBuffers.add(new ManagedBuffer(AUX_BUFFER_SIZE));
i++;
}
ManagedBuffer aux = auxiliaryBuffers.get(i);
available = aux.available();
if (available < 1) {
// secondary auxiliary buffer
available = len;
aux = new ManagedBuffer(Math.max(len, AUX_BUFFER_SIZE2));
auxiliaryBuffers.add(aux);
i++;
} else {
available = Math.min(len, available);
}
aux.write(b, off, (int) available);
len -= available;
if (len > 0) off += available;
}
}
}
private void writeOutside(byte buffer[], int offset, int length) throws IOException {
out.write(buffer, offset, length);
position += length;
if (onProgress != null && position > reportPosition) {
reportPosition = position + NOTIFY_BYTES_INTERVAL;
onProgress.report(position);
}
}
private void writeQueue(byte[] buffer, int offset, int length) throws IOException {
while (length > 0) {
if (queue.available() < length) {
flushQueue();
if (length >= queue.buffer.length) {
writeOutside(buffer, offset, length);
return;
}
}
int size = Math.min(queue.available(), length);
queue.write(buffer, offset, size);
offset += size;
length -= size;
}
if (queue.size >= queue.buffer.length) {
flushQueue();
}
}
private void flushQueue() throws IOException {
writeOutside(queue.buffer, 0, queue.size);
queue.size = 0;
}
private void flushEverything() throws IOException {
flushQueue();
if (auxiliaryBuffers.size() > 0) {
for (ManagedBuffer aux : auxiliaryBuffers) {
writeOutside(aux.buffer, 0, aux.size);
aux.dereference();
}
auxiliaryBuffers.clear();
}
}
/**
* Flush any buffer directly to the file. Warning: use this method ONLY if
* all read dependencies are disposed
*
* @throws IOException if the dependencies are not disposed
*/
@Override
public void flush() throws IOException {
if (callback.check() != -1) {
throw new IOException("All read dependencies of this file must be disposed first");
}
flushEverything();
// Save the current file length in case the method {@code rewind()} is called
if (position > maxLengthKnown) {
maxLengthKnown = position;
}
}
@Override
public void rewind() throws IOException {
flush();
out.seek(startOffset);
if (onProgress != null) {
onProgress.report(-position);
}
position = startOffset;
reportPosition = startOffset;
}
@Override
public long skip(long amount) throws IOException {
flush();
position += amount;
out.seek(position);
return amount;
}
@Override
public boolean isDisposed() {
return out == null;
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canWrite() {
return true;
}
//<editor-fold defaultState="collapsed" desc="stub read methods">
@Override
public boolean canRead() {
return false;
}
@Override
public int read() {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer) {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer, int offset, int count) {
throw new UnsupportedOperationException("write-only");
}
@Override
public int available() {
throw new UnsupportedOperationException("write-only");
}
//</editor-fold>
public interface OffsetChecker {
/**
* Checks the amount of available space ahead
*
* @return absolute offset in the file where no more data SHOULD NOT be
* written. If the value is -1 the whole file will be used
*/
long check();
}
public interface ProgressReport {
void report(long progress);
}
class ManagedBuffer {
byte[] buffer;
int size;
ManagedBuffer(int length) {
buffer = new byte[length];
}
void dereference() {
buffer = null;
size = 0;
}
void dereference(int amount) {
if (amount > size) {
throw new IndexOutOfBoundsException("Invalid dereference amount (" + amount + ">=" + size + ")");
}
size -= amount;
System.arraycopy(buffer, amount, buffer, 0, size);
}
protected int available() {
return buffer.length - size;
}
private void write(byte[] b, int off, int len) {
System.arraycopy(b, off, buffer, size, len);
size += len;
}
@Override
public String toString() {
return "holding: " + String.valueOf(size) + " length: " + String.valueOf(buffer.length) + " available: " + String.valueOf(available());
}
}
}

View file

@ -0,0 +1,459 @@
package us.shandian.giga.postprocessing.io;
import android.support.annotation.NonNull;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
public class CircularFileWriter extends SharpStream {
private final static int QUEUE_BUFFER_SIZE = 8 * 1024;// 8 KiB
private final static int NOTIFY_BYTES_INTERVAL = 64 * 1024;// 64 KiB
private final static int THRESHOLD_AUX_LENGTH = 3 * 1024 * 1024;// 3 MiB
private OffsetChecker callback;
public ProgressReport onProgress;
public WriteErrorHandle onWriteError;
private long reportPosition;
private long maxLengthKnown = -1;
private BufferedFile out;
private BufferedFile aux;
public CircularFileWriter(File source, File temp, OffsetChecker checker) throws IOException {
if (checker == null) {
throw new NullPointerException("checker is null");
}
if (!temp.exists()) {
if (!temp.createNewFile()) {
throw new IOException("Cannot create a temporal file");
}
}
aux = new BufferedFile(temp);
out = new BufferedFile(source);
callback = checker;
reportPosition = NOTIFY_BYTES_INTERVAL;
}
private void flushAuxiliar() throws IOException {
if (aux.length < 1) {
return;
}
boolean underflow = out.getOffset() >= out.length;
out.flush();
aux.flush();
aux.target.seek(0);
out.target.seek(out.length);
long length = aux.length;
out.length += aux.length;
while (length > 0) {
int read = (int) Math.min(length, Integer.MAX_VALUE);
read = aux.target.read(aux.queue, 0, Math.min(read, aux.queue.length));
out.writeProof(aux.queue, read);
length -= read;
}
if (underflow) {
out.offset += aux.offset;
out.target.seek(out.offset);
} else {
out.offset = out.length;
}
if (out.length > maxLengthKnown) {
maxLengthKnown = out.length;
}
if (aux.length > THRESHOLD_AUX_LENGTH) {
aux.target.setLength(THRESHOLD_AUX_LENGTH);// or setLength(0);
}
aux.reset();
}
/**
* Flush any buffer and close the output file. Use this method if the
* operation is successful
*
* @return the final length of the file
* @throws IOException if an I/O error occurs
*/
public long finalizeFile() throws IOException {
flushAuxiliar();
out.flush();
// change file length (if required)
long length = Math.max(maxLengthKnown, out.length);
if (length != out.target.length()) {
out.target.setLength(length);
}
dispose();
return length;
}
/**
* Close the file without flushing any buffer
*/
@Override
public void dispose() {
if (out != null) {
out.dispose();
out = null;
}
if (aux != null) {
aux.dispose();
aux = null;
}
}
@Override
public void write(byte b) throws IOException {
write(new byte[]{b}, 0, 1);
}
@Override
public void write(byte b[]) throws IOException {
write(b, 0, b.length);
}
@Override
public void write(byte b[], int off, int len) throws IOException {
if (len == 0) {
return;
}
long available;
long offsetOut = out.getOffset();
long offsetAux = aux.getOffset();
long end = callback.check();
if (end == -1) {
available = Integer.MAX_VALUE;
} else if (end < offsetOut) {
throw new IOException("The reported offset is invalid: " + String.valueOf(offsetOut));
} else {
available = end - offsetOut;
}
boolean usingAux = aux.length > 0 && offsetOut >= out.length;
boolean underflow = offsetAux < aux.length || offsetOut < out.length;
if (usingAux) {
// before continue calculate the final length of aux
long length = offsetAux + len;
if (underflow) {
if (aux.length > length) {
length = aux.length;// the length is not changed
}
} else {
length = aux.length + len;
}
if (length > available || length < THRESHOLD_AUX_LENGTH) {
aux.write(b, off, len);
} else {
if (underflow) {
aux.write(b, off, len);
flushAuxiliar();
} else {
flushAuxiliar();
out.write(b, off, len);// write directly on the output
}
}
} else {
if (underflow) {
available = out.length - offsetOut;
}
int length = Math.min(len, (int) available);
out.write(b, off, length);
len -= length;
off += length;
if (len > 0) {
aux.write(b, off, len);
}
}
if (onProgress != null) {
long absoluteOffset = out.getOffset() + aux.getOffset();
if (absoluteOffset > reportPosition) {
reportPosition = absoluteOffset + NOTIFY_BYTES_INTERVAL;
onProgress.report(absoluteOffset);
}
}
}
@Override
public void flush() throws IOException {
aux.flush();
out.flush();
long total = out.length + aux.length;
if (total > maxLengthKnown) {
maxLengthKnown = total;// save the current file length in case the method {@code rewind()} is called
}
}
@Override
public long skip(long amount) throws IOException {
seek(out.getOffset() + aux.getOffset() + amount);
return amount;
}
@Override
public void rewind() throws IOException {
if (onProgress != null) {
onProgress.report(-out.length - aux.length);// rollback the whole progress
}
seek(0);
reportPosition = NOTIFY_BYTES_INTERVAL;
}
@Override
public void seek(long offset) throws IOException {
long total = out.length + aux.length;
if (offset == total) {
return;// nothing to do
}
// flush everything, avoid any underflow
flush();
if (offset < 0 || offset > total) {
throw new IOException("desired offset is outside of range=0-" + total + " offset=" + offset);
}
if (offset > out.length) {
out.seek(out.length);
aux.seek(offset - out.length);
} else {
out.seek(offset);
aux.seek(0);
}
}
@Override
public boolean isDisposed() {
return out == null;
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canWrite() {
return true;
}
@Override
public boolean canSeek() {
return true;
}
// <editor-fold defaultstate="collapsed" desc="stub read methods">
@Override
public boolean canRead() {
return false;
}
@Override
public int read() {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer
) {
throw new UnsupportedOperationException("write-only");
}
@Override
public int read(byte[] buffer, int offset, int count
) {
throw new UnsupportedOperationException("write-only");
}
@Override
public long available() {
throw new UnsupportedOperationException("write-only");
}
//</editor-fold>
public interface OffsetChecker {
/**
* Checks the amount of available space ahead
*
* @return absolute offset in the file where no more data SHOULD NOT be
* written. If the value is -1 the whole file will be used
*/
long check();
}
public interface ProgressReport {
/**
* Report the size of the new file
*
* @param progress the new size
*/
void report(long progress);
}
public interface WriteErrorHandle {
/**
* Attempts to handle a I/O exception
*
* @param err the cause
* @return {@code true} to retry and continue, otherwise, {@code false}
* and throw the exception
*/
boolean handle(Exception err);
}
class BufferedFile {
protected final RandomAccessFile target;
private long offset;
protected long length;
private byte[] queue;
private int queueSize;
BufferedFile(File file) throws FileNotFoundException {
queue = new byte[QUEUE_BUFFER_SIZE];
target = new RandomAccessFile(file, "rw");
}
protected long getOffset() {
return offset + queueSize;// absolute offset in the file
}
protected void dispose() {
try {
queue = null;
target.close();
} catch (IOException e) {
// nothing to do
}
}
protected void write(byte b[], int off, int len) throws IOException {
while (len > 0) {
// if the queue is full, the method available() will flush the queue
int read = Math.min(available(), len);
// enqueue incoming buffer
System.arraycopy(b, off, queue, queueSize, read);
queueSize += read;
len -= read;
off += read;
}
long total = offset + queueSize;
if (total > length) {
length = total;// save length
}
}
protected void flush() throws IOException {
writeProof(queue, queueSize);
offset += queueSize;
queueSize = 0;
}
protected void rewind() throws IOException {
offset = 0;
target.seek(0);
}
protected int available() throws IOException {
if (queueSize >= queue.length) {
flush();
return queue.length;
}
return queue.length - queueSize;
}
protected void reset() throws IOException {
offset = 0;
length = 0;
target.seek(0);
}
protected void seek(long absoluteOffset) throws IOException {
offset = absoluteOffset;
target.seek(absoluteOffset);
}
protected void writeProof(byte[] buffer, int length) throws IOException {
if (onWriteError == null) {
target.write(buffer, 0, length);
return;
}
while (true) {
try {
target.write(buffer, 0, length);
return;
} catch (Exception e) {
if (!onWriteError.handle(e)) {
throw e;// give up
}
}
}
}
@NonNull
@Override
public String toString() {
String absOffset;
String absLength;
try {
absOffset = Long.toString(target.getFilePointer());
} catch (IOException e) {
absOffset = "[" + e.getLocalizedMessage() + "]";
}
try {
absLength = Long.toString(target.length());
} catch (IOException e) {
absLength = "[" + e.getLocalizedMessage() + "]";
}
return String.format(
"offset=%s length=%s queue=%s absOffset=%s absLength=%s",
offset, length, queueSize, absOffset, absLength
);
}
}
}

View file

@ -1,126 +0,0 @@
package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
/**
* @author kapodamy
*/
public class FileStream extends SharpStream {
public enum Mode {
Read,
ReadWrite
}
public RandomAccessFile source;
private final Mode mode;
public FileStream(String path, Mode mode) throws IOException {
String flags;
if (mode == Mode.Read) {
flags = "r";
} else {
flags = "rw";
}
this.mode = mode;
source = new RandomAccessFile(path, flags);
}
@Override
public int read() throws IOException {
return source.read();
}
@Override
public int read(byte b[]) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte b[], int off, int len) throws IOException {
return source.read(b, off, len);
}
@Override
public long skip(long pos) throws IOException {
FileChannel fc = source.getChannel();
fc.position(fc.position() + pos);
return pos;
}
@Override
public int available() {
try {
return (int) (source.length() - source.getFilePointer());
} catch (IOException ex) {
return 0;
}
}
@SuppressWarnings("EmptyCatchBlock")
@Override
public void dispose() {
try {
source.close();
} catch (IOException err) {
} finally {
source = null;
}
}
@Override
public boolean isDisposed() {
return source == null;
}
@Override
public void rewind() throws IOException {
source.getChannel().position(0);
}
@Override
public boolean canRewind() {
return true;
}
@Override
public boolean canRead() {
return mode == Mode.Read || mode == Mode.ReadWrite;
}
@Override
public boolean canWrite() {
return mode == Mode.ReadWrite;
}
@Override
public void write(byte value) throws IOException {
source.write(value);
}
@Override
public void write(byte[] buffer) throws IOException {
source.write(buffer);
}
@Override
public void write(byte[] buffer, int offset, int count) throws IOException {
source.write(buffer, offset, count);
}
@Override
public void flush() {
}
@Override
public void setLength(long length) throws IOException {
source.setLength(length);
}
}

View file

@ -14,6 +14,7 @@ import java.io.InputStream;
/**
* Wrapper for the classic {@link java.io.InputStream}
*
* @author kapodamy
*/
public class SharpInputStream extends InputStream {
@ -49,7 +50,8 @@ public class SharpInputStream extends InputStream {
@Override
public int available() {
return base.available();
long res = base.available();
return res > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) res;
}
@Override