Sulkta-OSS/straw
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update extractor and refactored NewPipe

Browse source
This commit is contained in:
Mauricio Colli 2017-09-03 03:04:18 -03:00
parent bddd9b3409
commit 146d4a8365
254 changed files with 7836 additions and 7302 deletions
Download patch file Download diff file Expand all files Collapse all files

101
app/src/main/java/org/schabi/newpipe/Downloader.java
View file

@ -1,20 +1,24 @@
package org.schabi.newpipe;
import android.util.Log;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.util.ExtractorHelper;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.net.ssl.HttpsURLConnection;
/**
/*
* Created by Christian Schabesberger on 28.01.16.
*
* Copyright (C) Christian Schabesberger 2016 <chris.schabesberger@mailbox.org>
@ -35,16 +39,17 @@ import javax.net.ssl.HttpsURLConnection;
*/
public class Downloader implements org.schabi.newpipe.extractor.Downloader {
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0";
private static String mCookies = "";
private static Downloader instance = null;
private Downloader() {}
private Downloader() {
}
public static Downloader getInstance() {
if(instance == null) {
if (instance == null) {
synchronized (Downloader.class) {
if (instance == null) {
instance = new Downloader();
@ -62,41 +67,66 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
return Downloader.mCookies;
}
/**Download the text file at the supplied URL as in download(String),
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
*
* @param siteUrl the URL of the text file to return the contents of
* @param language the language (usually a 2-character code) to set as the preferred language
* @return the contents of the specified text file*/
* @return the contents of the specified text file
*/
@Override
public String download(String siteUrl, String language) throws IOException, ReCaptchaException {
Map<String, String> requestProperties = new HashMap<>();
requestProperties.put("Accept-Language", language);
return download(siteUrl, requestProperties);
}
/**Download the text file at the supplied URL as in download(String),
* but set the HTTP header field "Accept-Language" to the supplied string.
* @param siteUrl the URL of the text file to return the contents of
/**
* Download the text file at the supplied URL as in download(String),
* but set the HTTP headers included in the customProperties map.
*
* @param siteUrl the URL of the text file to return the contents of
* @param customProperties set request header properties
* @return the contents of the specified text file
* @throws IOException*/
* @throws IOException
*/
@Override
public String download(String siteUrl, Map<String, String> customProperties) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
Iterator it = customProperties.entrySet().iterator();
while(it.hasNext()) {
Map.Entry pair = (Map.Entry)it.next();
con.setRequestProperty((String)pair.getKey(), (String)pair.getValue());
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
con.setRequestProperty((String) pair.getKey(), (String) pair.getValue());
}
return dl(con);
}
/**Common functionality between download(String url) and download(String url, String language)*/
/**
* Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
*
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file
*/
@Override
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
/**
* Common functionality between download(String url) and download(String url, String language)
*/
private static String dl(HttpsURLConnection con) throws IOException, ReCaptchaException {
StringBuilder response = new StringBuilder();
BufferedReader in = null;
try {
con.setReadTimeout(30 * 1000);// 30s
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
@ -104,17 +134,22 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
con.setRequestProperty("Cookie", getCookies());
}
in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
in = new BufferedReader(new InputStreamReader(con.getInputStream()));
for (Map.Entry<String, List<String>> entry : con.getHeaderFields().entrySet()) {
System.err.println(entry.getKey() + ": " + entry.getValue());
}
String inputLine;
while((inputLine = in.readLine()) != null) {
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
} catch(UnknownHostException uhe) {//thrown when there's no internet connection
throw new IOException("unknown host or no network", uhe);
//Toast.makeText(getActivity(), uhe.getMessage(), Toast.LENGTH_LONG).show();
} catch(Exception e) {
} catch (Exception e) {
Log.e("Downloader", "dl() ----- Exception thrown → " + e.getClass().getName());
if (ExtractorHelper.isInterruptedCaused(e)) {
throw new InterruptedIOException(e.getMessage());
}
/*
* HTTP 429 == Too Many Request
* Receive from Youtube.com = ReCaptcha challenge request
@ -123,24 +158,14 @@ public class Downloader implements org.schabi.newpipe.extractor.Downloader {
if (con.getResponseCode() == 429) {
throw new ReCaptchaException("reCaptcha Challenge requested");
}
throw new IOException(e);
throw new IOException(con.getResponseCode() + " " + con.getResponseMessage(), e);
} finally {
if(in != null) {
if (in != null) {
in.close();
}
}
return response.toString();
}
/**Download (via HTTP) the text file located at the supplied URL, and return its contents.
* Primarily intended for downloading web pages.
* @param siteUrl the URL of the text file to download
* @return the contents of the specified text file*/
public String download(String siteUrl) throws IOException, ReCaptchaException {
URL url = new URL(siteUrl);
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
//HttpsURLConnection con = NetCipher.getHttpsURLConnection(url);
return dl(con);
}
}