From bbaa6cdb902528200708b950a8858cb350ec3796 Mon Sep 17 00:00:00 2001 From: ThetaDev Date: Wed, 12 Oct 2022 23:53:48 +0200 Subject: [PATCH] feat: add fallback to player query --- README.md | 1 + cli/src/main.rs | 17 +-- codegen/src/download_testfiles.rs | 11 +- notes/video_ids.txt | 5 +- src/client/player.rs | 24 +++- src/model/mod.rs | 2 + tests/youtube.rs | 175 +++++++++++++++++++++++++++--- 7 files changed, 204 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 8d0d4f6..bde5689 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ inspired by [NewPipe](https://github.com/TeamNewPipe/NewPipeExtractor). - [X] **Search** (with filters) - [ ] **Search suggestions** - [ ] **Trending** +- [ ] **URL resolver** ### YouTube Music diff --git a/cli/src/main.rs b/cli/src/main.rs index 53ab613..ec603f8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -5,10 +5,7 @@ use clap::{Parser, Subcommand}; use futures::stream::{self, StreamExt}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use reqwest::{Client, ClientBuilder}; -use rustypipe::{ - client::{ClientType, RustyPipe}, - param::StreamFilter, -}; +use rustypipe::{client::RustyPipe, param::StreamFilter}; #[derive(Parser)] #[clap(author, version, about, long_about = None)] @@ -58,14 +55,10 @@ async fn download_single_video( pb.set_message(format!("Fetching player data for {}", video_title)); let res = async { - let player_data = rp - .query() - .player(video_id.as_str(), ClientType::TvHtml5Embed) - .await - .context(format!( - "Failed to fetch player data for video {}", - video_id - ))?; + let player_data = rp.query().player(video_id.as_str()).await.context(format!( + "Failed to fetch player data for video {}", + video_id + ))?; let mut filter = StreamFilter::default(); if let Some(res) = resolution { diff --git a/codegen/src/download_testfiles.rs b/codegen/src/download_testfiles.rs index 3b63f97..0b6b85a 100644 --- a/codegen/src/download_testfiles.rs +++ b/codegen/src/download_testfiles.rs @@ -89,7 +89,10 @@ async fn player(testfiles: &Path) { } let rp = rp_testfile(&json_path); - rp.query().player(video_id, client_type).await.unwrap(); + rp.query() + .player_from_client(video_id, client_type) + .await + .unwrap(); } } @@ -105,7 +108,11 @@ async fn player_model(testfiles: &Path) { continue; } - let player_data = rp.query().player(id, ClientType::Desktop).await.unwrap(); + let player_data = rp + .query() + .player_from_client(id, ClientType::Desktop) + .await + .unwrap(); let file = File::create(&json_path).unwrap(); serde_json::to_writer_pretty(file, &player_data).unwrap(); diff --git a/notes/video_ids.txt b/notes/video_ids.txt index fd29b57..53f9d7b 100644 --- a/notes/video_ids.txt +++ b/notes/video_ids.txt @@ -1,4 +1,4 @@ -CC Video: pPvd8UxmSbQ +CCommons Video: pPvd8UxmSbQ Video: ZeerrnuLi5E 4K HDR: LXb3EKWsInQ 8K: Zv11L-ZfrSg @@ -6,7 +6,7 @@ Music: ihUZMeYFZHA Multilanguage: tVWWp1PqDus # Livestreams -Live: 64DYi_8ESh0 +Live: 86YLFOog4GM Was live: pxY4OXVyMe4 # Errors @@ -15,6 +15,7 @@ Censored: 6SJNVb0GnPI Geoblocked: sJL6WA-aGkQ (Japan only) Private: s7_qI6_mIXc DRM: 1bfOsni7EgI +Deleted: 64DYi_8ESh0 Album with unknown artists: https://music.youtube.com/playlist?list=OLAK5uy_mEX9ljZeeEWgTM1xLL1isyiGaWXoPyoOk diff --git a/src/client/player.rs b/src/client/player.rs index 87543d9..a22e4c0 100644 --- a/src/client/player.rs +++ b/src/client/player.rs @@ -58,7 +58,23 @@ struct QContentPlaybackContext { } impl RustyPipeQuery { - pub async fn player( + pub async fn player(self, video_id: &str) -> Result { + let q1 = self.clone(); + let android_res = q1.player_from_client(video_id, ClientType::Android).await; + + match android_res { + Ok(res) => Ok(res), + Err(Error::Extraction( + ExtractionError::VideoAgeRestricted | ExtractionError::WrongResult(_), + )) => { + self.player_from_client(video_id, ClientType::TvHtml5Embed) + .await + } + Err(e) => Err(e), + } + } + + pub async fn player_from_client( self, video_id: &str, client_type: ClientType, @@ -129,7 +145,11 @@ impl MapResponse for response::Player { } response::player::PlayabilityStatus::LoginRequired { reason } => { // reason: "Sign in to confirm your age" - if reason.split_whitespace().any(|word| word == "age") { + // or: "This video may be inappropriate for some users." + if reason + .split_whitespace() + .any(|word| word == "age" || word == "inappropriate") + { return Err(ExtractionError::VideoAgeRestricted); } return Err(ExtractionError::VideoUnavailable("private video", reason)); diff --git a/src/model/mod.rs b/src/model/mod.rs index 883ee27..1051122 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -66,6 +66,8 @@ pub struct VideoPlayerDetails { /// Video description in plaintext format pub description: Option, /// Video length in seconds + /// + /// Is zero for livestreams pub length: u32, /// Video thumbnail pub thumbnail: Vec, diff --git a/tests/youtube.rs b/tests/youtube.rs index 9a92272..24a519b 100644 --- a/tests/youtube.rs +++ b/tests/youtube.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use chrono::Datelike; use rstest::rstest; @@ -20,9 +22,13 @@ use rustypipe::param::{ #[case::android(ClientType::Android)] #[case::ios(ClientType::Ios)] #[tokio::test] -async fn get_player(#[case] client_type: ClientType) { +async fn get_player_from_client(#[case] client_type: ClientType) { let rp = RustyPipe::builder().strict().build(); - let player_data = rp.query().player("n4tK7LYFxI0", client_type).await.unwrap(); + let player_data = rp + .query() + .player_from_client("n4tK7LYFxI0", client_type) + .await + .unwrap(); // dbg!(&player_data); @@ -39,7 +45,7 @@ async fn get_player(#[case] client_type: ClientType) { assert!(!player_data.details.thumbnail.is_empty()); assert_eq!(player_data.details.channel.id, "UC_aEa8K-EOJ3D6gOs7HcyNg"); assert_eq!(player_data.details.channel.name, "NoCopyrightSounds"); - assert!(player_data.details.view_count > 146818808); + assert!(player_data.details.view_count > 146_818_808); assert_eq!(player_data.details.keywords[0], "spektrem"); assert_eq!(player_data.details.is_live_content, false); @@ -111,20 +117,163 @@ async fn get_player(#[case] client_type: ClientType) { assert!(player_data.expires_in_seconds > 10000); } -/* #[rstest] -#[case::desktop(ClientType::Desktop)] -// #[case::tv_html5_embed(ClientType::TvHtml5Embed)] -// #[case::android(ClientType::Android)] -// #[case::ios(ClientType::Ios)] -#[test_log::test(tokio::test)] -async fn get_player_live(#[case] client_type: ClientType) { +#[case::music( + "ihUZMeYFZHA", + "Oonagh - Nan Úye", + "Offizielle AlbumPlaylist:", + 260, + "UC2llNlEM62gU-_fXPHfgbDg", + "Oonagh", + 830900, + false, + false +)] +#[case::hdr( + "LXb3EKWsInQ", + "COSTA RICA IN 4K 60fps HDR (ULTRA HD)", + "We've re-mastered and re-uploaded our favorite video in HDR!", + 314, + "UCYq-iAOSZBvoUxvfzwKIZWA", + "Jacob + Katie Schwarz", + 220_000_000, + false, + false +)] +#[case::multilanguage( + "tVWWp1PqDus", + "100 Girls Vs 100 Boys For $500,000", + "Giving away $25k on Current!", + 1013, + "UCX6OQ3DkcsbYNE6H8uQQuVA", + "MrBeast", + 82_000_000, + false, + false +)] +#[case::live( + "86YLFOog4GM", + "🌎 Nasa Live Stream - Earth From Space : Live Views from the ISS", + "Live NASA - Views Of Earth from Space", + 0, + "UCakgsb0w7QB0VHdnCc-OVEA", + "Space Videos", + 10, + true, + true +)] +#[case::was_live( + "pxY4OXVyMe4", + "Minecraft GENESIS LIVESTREAM!!", + "FÜR MEHR LIVESTREAMS AUF YOUTUBE MACHT FOLGENDES", + 5535, + "UCQM0bS4_04-Y4JuYrgmnpZQ", + "Chaosflo44", + 500_000, + false, + true +)] +#[case::agelimit( + "laru0QoJUmI", + "DJ Robin x Schürze - Layla (Official Video)", + "Endlich ist es soweit! Zwei Männer aus dem Schwabenland", + 188, + "UCkJfSrMnLonOZWh-q5os5bg", + "Summerfield Records", + 10_000_000, + false, + false +)] +#[tokio::test] +async fn get_player( + #[case] id: &str, + #[case] title: &str, + #[case] description: &str, + #[case] length: u32, + #[case] channel_id: &str, + #[case] channel_name: &str, + #[case] views: u64, + #[case] is_live: bool, + #[case] is_live_content: bool, +) { let rp = RustyPipe::builder().strict().build(); - let player_data = rp.query().player("86YLFOog4GM", client_type).await.unwrap(); + let player_data = rp.query().player(id).await.unwrap(); + let details = player_data.details; - dbg!(&player_data); + assert_eq!(details.id, id); + assert_eq!(details.title, title); + let desc = details.description.unwrap(); + assert!(desc.contains(description), "description: {}", desc); + assert_eq!(details.length, length); + assert_eq!(details.channel.id, channel_id); + assert_eq!(details.channel.name, channel_name); + assert!( + details.view_count > views, + "expected > {} views, got {}", + views, + details.view_count + ); + assert_eq!(details.is_live, is_live); + assert_eq!(details.is_live_content, is_live_content); + + if is_live { + assert!(player_data.hls_manifest_url.is_some()); + assert!(player_data.dash_manifest_url.is_some()); + } else { + assert!(!player_data.video_only_streams.is_empty()); + assert!(!player_data.audio_streams.is_empty()); + } + + match id { + // HDR + "LXb3EKWsInQ" => { + assert!( + player_data + .video_only_streams + .iter() + .any(|stream| stream.hdr), + "no hdr streams" + ); + } + // Multilanguage + "tVWWp1PqDus" => { + let langs = player_data + .audio_streams + .iter() + .filter_map(|stream| { + stream + .track + .as_ref() + .map(|t| t.lang.as_ref().unwrap().to_owned()) + }) + .collect::>(); + + for l in ["en", "es", "fr", "pt", "ru"] { + assert!(langs.contains(l), "missing lang: {}", l); + } + } + _ => {} + }; + + assert!(player_data.expires_in_seconds > 10000); +} + +#[rstest] +#[case::not_found("86abcdefghi", "extraction error: Video cant be played because of deletion/censorship. Reason (from YT): This video is unavailable")] +#[case::deleted("64DYi_8ESh0", "extraction error: Video cant be played because of deletion/censorship. Reason (from YT): This video is unavailable")] +#[case::censored("6SJNVb0GnPI", "extraction error: Video cant be played because of deletion/censorship. Reason (from YT): This video has been removed for violating YouTube's policy on hate speech. Learn more about combating hate speech in your country.")] +// This video is geoblocked outside of Japan, so expect this test case to fail when using a Japanese IP address. +#[case::geoblock("sJL6WA-aGkQ", "extraction error: Video cant be played because of DRM/Geoblock. Reason (from YT): The uploader has not made this video available in your country")] +#[case::drm("1bfOsni7EgI", "extraction error: Video cant be played because of DRM/Geoblock. Reason (from YT): This video can only be played on newer versions of Android or other supported devices.")] +#[case::private("s7_qI6_mIXc", "extraction error: Video cant be played because of private video. Reason (from YT): This video is private")] +#[case::t1("CUO8secmc0g", "extraction error: Video cant be played because of DRM/Geoblock. Reason (from YT): Playback on other websites has been disabled by the video owner")] +#[tokio::test] +async fn get_player_error(#[case] id: &str, #[case] msg: &str) { + let rp = RustyPipe::builder().strict().build(); + let err = rp.query().player(id).await.unwrap_err(); + + assert_eq!(err.to_string(), msg); } -*/ //#PLAYLIST