Cobb
b231c84d9a
All checks were successful
gitleaks / scan (push) Successful in 8s
The planted badge in a_badge_with_digits_that_is_not_a_clock_is_listed_as_past was "28h10". It has no ":", so parse_lockup_video never stored a duration and deleting the reset to 0 on that branch changed nothing: the test passed with it deleted. With "28:1" the parser stores 1681 seconds first, and the test now fails with left 1681, right 0 when the reset is deleted. The test still passes as written, and the full suite passes 225 of 225.
790 lines
28 KiB
Rust
790 lines
28 KiB
Rust
// A channel's Live tab, pinned against responses captured from public
|
|
// channels on 2026-10-23 (tests/fixtures/live_tab/README.md). Offline: the
|
|
// fetch tests install a Downloader that serves the fixtures.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use serde_json::Value;
|
|
use strawcore_core::exceptions::NetworkError;
|
|
use strawcore_core::stream::StreamType;
|
|
use strawcore_core::youtube::channel_live::{
|
|
channel_live, channel_live_continuation, has_live_tab, parse_live_tab,
|
|
parse_live_tab_continuation, LiveStatus, LiveTabItem, LiveTabPage,
|
|
};
|
|
use strawcore_core::{ContentCountry, Downloader, Localization, NewPipe, Request, Response};
|
|
|
|
const NASA_LIVE: &str = include_str!("fixtures/live_tab/nasa_live.json");
|
|
const NASA_LIVE_PAGE2: &str = include_str!("fixtures/live_tab/nasa_live_page2.json");
|
|
const SKYNEWS_LIVE_PAGE2: &str = include_str!("fixtures/live_tab/skynews_live_page2.json");
|
|
const SKYNEWS_LIVE: &str = include_str!("fixtures/live_tab/skynews_live.json");
|
|
const SKYNEWS_LIVE_DE: &str = include_str!("fixtures/live_tab/skynews_live_de.json");
|
|
const LOFIGIRL_LIVE: &str = include_str!("fixtures/live_tab/lofigirl_live.json");
|
|
const LOFIGIRL_LIVE_DE: &str = include_str!("fixtures/live_tab/lofigirl_live_de.json");
|
|
const LOFIGIRL_LIVE_FR: &str = include_str!("fixtures/live_tab/lofigirl_live_fr.json");
|
|
const LOFIGIRL_LIVE_FI: &str = include_str!("fixtures/live_tab/lofigirl_live_fi.json");
|
|
const LOFIGIRL_LIVE_ID: &str = include_str!("fixtures/live_tab/lofigirl_live_id.json");
|
|
const NASA_LIVE_DA: &str = include_str!("fixtures/live_tab/nasa_live_da.json");
|
|
const SPACEX_LIVE: &str = include_str!("fixtures/live_tab/spacex_live.json");
|
|
const NASASPACEFLIGHT_LIVE: &str = include_str!("fixtures/live_tab/nasaspaceflight_live.json");
|
|
const NASA_HOME_TABS: &str = include_str!("fixtures/live_tab/nasa_home_tabs.json");
|
|
const KURZGESAGT_LIVE_REQUEST: &str =
|
|
include_str!("fixtures/live_tab/kurzgesagt_live_request.json");
|
|
|
|
const NASA_ID: &str = "UCLA_DiR1FfKNvjuUpBHmylQ";
|
|
const LIVE_PARAMS: &str = "EgdzdHJlYW1z8gYECgJ6AA==";
|
|
|
|
fn json(text: &str) -> Value {
|
|
serde_json::from_str(text).expect("fixture is JSON")
|
|
}
|
|
|
|
fn en_gb() -> Localization {
|
|
Localization::default()
|
|
}
|
|
|
|
fn lang(code: &str) -> Localization {
|
|
Localization::new(code, None)
|
|
}
|
|
|
|
fn page(text: &str, localization: &Localization) -> LiveTabPage {
|
|
parse_live_tab(&json(text), localization)
|
|
}
|
|
|
|
/// One letter per item: S scheduled, L live now, P past.
|
|
fn kinds(page: &LiveTabPage) -> String {
|
|
page.items
|
|
.iter()
|
|
.map(|i| match i.status {
|
|
LiveStatus::Scheduled { .. } => 'S',
|
|
LiveStatus::LiveNow { .. } => 'L',
|
|
LiveStatus::Past => 'P',
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn run(s: usize, l: usize, p: usize) -> String {
|
|
format!("{}{}{}", "S".repeat(s), "L".repeat(l), "P".repeat(p))
|
|
}
|
|
|
|
fn watch(id: &str) -> String {
|
|
format!("https://www.youtube.com/watch?v={id}")
|
|
}
|
|
|
|
#[test]
|
|
fn every_captured_item_is_kept_and_classified_in_the_order_given() {
|
|
let de = Localization::new("de", Some("DE".into()));
|
|
let fr = Localization::new("fr", Some("FR".into()));
|
|
for (name, text, localization, expected, more) in [
|
|
("nasa", NASA_LIVE, en_gb(), run(2, 2, 26), true),
|
|
("skynews", SKYNEWS_LIVE, en_gb(), run(3, 2, 25), true),
|
|
(
|
|
"skynews de",
|
|
SKYNEWS_LIVE_DE,
|
|
de.clone(),
|
|
run(3, 2, 25),
|
|
true,
|
|
),
|
|
("lofigirl", LOFIGIRL_LIVE, en_gb(), run(0, 23, 2), false),
|
|
("lofigirl de", LOFIGIRL_LIVE_DE, de, run(0, 23, 2), false),
|
|
("lofigirl fr", LOFIGIRL_LIVE_FR, fr, run(0, 23, 2), false),
|
|
("spacex", SPACEX_LIVE, en_gb(), run(0, 0, 30), true),
|
|
("nasa da", NASA_LIVE_DA, lang("da"), run(1, 3, 26), true),
|
|
(
|
|
"lofigirl fi",
|
|
LOFIGIRL_LIVE_FI,
|
|
lang("fi"),
|
|
run(0, 23, 2),
|
|
false,
|
|
),
|
|
(
|
|
"lofigirl id",
|
|
LOFIGIRL_LIVE_ID,
|
|
lang("id"),
|
|
run(0, 23, 2),
|
|
false,
|
|
),
|
|
(
|
|
"nasaspaceflight",
|
|
NASASPACEFLIGHT_LIVE,
|
|
en_gb(),
|
|
run(1, 3, 26),
|
|
true,
|
|
),
|
|
] {
|
|
let p = page(text, &localization);
|
|
assert_eq!(kinds(&p), expected, "{name}");
|
|
assert_eq!(p.continuation.is_some(), more, "{name} continuation");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_scheduled_stream_carries_its_audience_and_start() {
|
|
let p = page(NASA_LIVE, &en_gb());
|
|
let first = &p.items[0];
|
|
assert_eq!(first.item.url, watch("hv867HnY4j4"));
|
|
assert_eq!(first.item.name, "U.S. Spacewalk 100");
|
|
assert_eq!(
|
|
first.status,
|
|
LiveStatus::Scheduled {
|
|
waiting: Some("5 waiting".into()),
|
|
start_text: Some("Scheduled for 29/10/2026, 10:45".into()),
|
|
// The video's own scheduledStartTime, measured the same day.
|
|
start_time: Some(1_793_270_700),
|
|
}
|
|
);
|
|
assert_eq!(first.item.stream_type, Some(StreamType::VideoStream));
|
|
assert_eq!(first.item.view_count, -1);
|
|
assert_eq!(first.item.duration_seconds, 0);
|
|
assert_eq!(first.item.upload_date_relative, "");
|
|
assert_eq!(
|
|
first.item.uploader_name, "",
|
|
"\"5 waiting\" is not an uploader"
|
|
);
|
|
assert!(!first.item.thumbnails.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_scheduled_stream_without_a_waiting_count_still_has_its_start() {
|
|
let p = page(SKYNEWS_LIVE, &en_gb());
|
|
assert_eq!(p.items[0].item.url, watch("5h2gFN76HJc"));
|
|
assert_eq!(
|
|
p.items[0].status,
|
|
LiveStatus::Scheduled {
|
|
waiting: None,
|
|
start_text: Some("Scheduled for 24/10/2026, 05:00".into()),
|
|
start_time: Some(1_792_818_000),
|
|
}
|
|
);
|
|
assert_eq!(p.items[0].item.uploader_name, "");
|
|
}
|
|
|
|
#[test]
|
|
fn a_live_stream_carries_its_audience_and_is_typed_live() {
|
|
let p = page(NASA_LIVE, &en_gb());
|
|
let live = &p.items[2];
|
|
assert_eq!(live.item.url, watch("ROboUHfPUrU"));
|
|
assert_eq!(
|
|
live.status,
|
|
LiveStatus::LiveNow {
|
|
watching: Some("150 watching".into())
|
|
}
|
|
);
|
|
assert_eq!(live.item.stream_type, Some(StreamType::VideoLiveStream));
|
|
assert_eq!(live.item.view_count, -1);
|
|
assert_eq!(live.item.duration_seconds, 0);
|
|
assert_eq!(live.item.upload_date_relative, "");
|
|
assert_eq!(
|
|
live.item.uploader_name, "",
|
|
"\"150 watching\" is not an uploader"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_past_stream_carries_duration_views_and_age() {
|
|
let p = page(NASA_LIVE, &en_gb());
|
|
let past = &p.items[4];
|
|
assert_eq!(past.item.url, watch("UleiRdWFG7o"));
|
|
assert_eq!(past.status, LiveStatus::Past);
|
|
assert_eq!(past.item.stream_type, Some(StreamType::VideoStream));
|
|
assert_eq!(past.item.duration_seconds, 28 * 60 + 10);
|
|
assert_eq!(past.item.view_count, 75_000);
|
|
assert_eq!(past.item.upload_date_relative, "Streamed 2 days ago");
|
|
assert_eq!(past.item.uploader_name, "");
|
|
}
|
|
|
|
#[test]
|
|
fn durations_past_999_hours_read_the_same_in_every_language() {
|
|
let long_a = 20_772 * 3600 + 51 * 60 + 34;
|
|
let long_b = 20_843 * 3600 + 43 * 60 + 51;
|
|
for (name, text, localization) in [
|
|
("en", LOFIGIRL_LIVE, en_gb()),
|
|
(
|
|
"de",
|
|
LOFIGIRL_LIVE_DE,
|
|
Localization::new("de", Some("DE".into())),
|
|
),
|
|
(
|
|
"fr",
|
|
LOFIGIRL_LIVE_FR,
|
|
Localization::new("fr", Some("FR".into())),
|
|
),
|
|
("fi", LOFIGIRL_LIVE_FI, lang("fi")),
|
|
("id", LOFIGIRL_LIVE_ID, lang("id")),
|
|
] {
|
|
let p = page(text, &localization);
|
|
let durations: Vec<i64> = p.items[23..]
|
|
.iter()
|
|
.map(|i| i.item.duration_seconds)
|
|
.collect();
|
|
assert_eq!(durations, vec![long_a, long_b], "{name}");
|
|
assert_eq!(p.items[23].item.url, watch("DWcJFNfaw9c"), "{name}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn durations_written_with_dots_read_like_those_written_with_colons() {
|
|
// da writes "28.10" and "1.42.26" where en-GB writes "28:10" and
|
|
// "1:42:26". Both pages were captured on 2026-10-23; every past stream
|
|
// on both must have the same duration.
|
|
let da = page(NASA_LIVE_DA, &lang("da"));
|
|
let en = page(NASA_LIVE, &en_gb());
|
|
let mut compared = 0;
|
|
for item in da.items.iter().filter(|i| i.status == LiveStatus::Past) {
|
|
if let Some(same) = en.items.iter().find(|i| i.item.url == item.item.url) {
|
|
assert_eq!(same.status, LiveStatus::Past, "{}", item.item.url);
|
|
assert_eq!(
|
|
item.item.duration_seconds, same.item.duration_seconds,
|
|
"{}",
|
|
item.item.url
|
|
);
|
|
compared += 1;
|
|
}
|
|
}
|
|
assert_eq!(compared, 26);
|
|
let first_past = da
|
|
.items
|
|
.iter()
|
|
.find(|i| i.item.url == watch("UleiRdWFG7o"))
|
|
.expect("past stream");
|
|
assert_eq!(first_past.item.duration_seconds, 28 * 60 + 10);
|
|
assert_eq!(
|
|
first_past.item.upload_date_relative,
|
|
"Blev streamet for 2 dage siden"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_scheduled_stream_in_a_language_that_writes_dots_is_still_scheduled() {
|
|
let p = page(NASA_LIVE_DA, &lang("da"));
|
|
assert_eq!(p.items[0].item.url, watch("hv867HnY4j4"));
|
|
assert_eq!(
|
|
p.items[0].status,
|
|
LiveStatus::Scheduled {
|
|
waiting: Some("7 venter".into()),
|
|
start_text: Some("Planlagt til 29.10.2026, 10.45".into()),
|
|
start_time: None,
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_translated_live_badge_is_still_live() {
|
|
let p = page(
|
|
LOFIGIRL_LIVE_FR,
|
|
&Localization::new("fr", Some("FR".into())),
|
|
);
|
|
assert_eq!(
|
|
p.items[0].status,
|
|
LiveStatus::LiveNow {
|
|
watching: Some("5,6\u{a0}k\u{a0}spectateurs".into())
|
|
}
|
|
);
|
|
assert_eq!(
|
|
p.items[0].item.stream_type,
|
|
Some(StreamType::VideoLiveStream)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_scheduled_start_in_another_language_keeps_its_text_but_no_time() {
|
|
let p = page(SKYNEWS_LIVE_DE, &Localization::new("de", Some("DE".into())));
|
|
assert_eq!(
|
|
p.items[1].status,
|
|
LiveStatus::Scheduled {
|
|
waiting: Some("42\u{a0}warten".into()),
|
|
start_text: Some("Geplant f\u{fc}r: 23.10.26, 21:00".into()),
|
|
start_time: None,
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn the_start_time_follows_the_request_language_not_the_text_alone() {
|
|
// The en-GB wording read under another localization is not trusted.
|
|
let p = page(NASA_LIVE, &Localization::new("en", Some("US".into())));
|
|
match &p.items[0].status {
|
|
LiveStatus::Scheduled {
|
|
start_time,
|
|
start_text,
|
|
..
|
|
} => {
|
|
assert_eq!(
|
|
start_text.as_deref(),
|
|
Some("Scheduled for 29/10/2026, 10:45")
|
|
);
|
|
assert_eq!(*start_time, None);
|
|
}
|
|
other => panic!("expected scheduled, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a_members_only_past_stream_has_an_age_and_no_view_count() {
|
|
let p = page(NASASPACEFLIGHT_LIVE, &en_gb());
|
|
let members: Vec<_> = p
|
|
.items
|
|
.iter()
|
|
.filter(|i| i.item.url == watch("aqyVrw3RjnU") || i.item.url == watch("Xowe-hUaeo8"))
|
|
.collect();
|
|
assert_eq!(members.len(), 2);
|
|
for m in members {
|
|
assert_eq!(m.status, LiveStatus::Past);
|
|
assert_eq!(m.item.view_count, -1);
|
|
assert_eq!(m.item.upload_date_relative, "Streamed 3 weeks ago");
|
|
assert!(m.item.duration_seconds > 0);
|
|
}
|
|
let fundraiser = p
|
|
.items
|
|
.iter()
|
|
.find(|i| i.item.url == watch("96X9Tt47ND0"))
|
|
.expect("fundraiser item");
|
|
assert_eq!(fundraiser.item.view_count, 47_000);
|
|
assert_eq!(fundraiser.item.upload_date_relative, "Streamed 4 days ago");
|
|
assert_eq!(fundraiser.item.duration_seconds, 3600 + 36 * 60 + 36);
|
|
}
|
|
|
|
#[test]
|
|
fn the_second_page_parses_from_the_continuation_response() {
|
|
let p = parse_live_tab_continuation(&json(NASA_LIVE_PAGE2), &en_gb());
|
|
assert_eq!(kinds(&p), run(0, 0, 30));
|
|
assert_eq!(p.items[0].item.url, watch("rSvCuSQhC3w"));
|
|
assert_eq!(p.items[0].item.duration_seconds, 3600 + 28 * 60 + 45);
|
|
assert_eq!(p.items[0].item.view_count, 3_400_000);
|
|
assert!(p.continuation.is_some());
|
|
}
|
|
|
|
#[test]
|
|
fn a_row_naming_collaborators_does_not_displace_views_and_age() {
|
|
// Captured: [["Sky News and Mornings With Ridge & Frost"],
|
|
// ["47k views", "Streamed 3 days ago"]].
|
|
let p = parse_live_tab_continuation(&json(SKYNEWS_LIVE_PAGE2), &en_gb());
|
|
assert_eq!(kinds(&p), run(0, 0, 30));
|
|
for (id, views, age, duration) in [
|
|
(
|
|
"fGRldHurmgg",
|
|
47_000,
|
|
"Streamed 3 days ago",
|
|
2 * 3600 + 56 * 60 + 35,
|
|
),
|
|
(
|
|
"un_Odvrj_kM",
|
|
33_000,
|
|
"Streamed 4 days ago",
|
|
2 * 3600 + 56 * 60 + 56,
|
|
),
|
|
] {
|
|
let item = p
|
|
.items
|
|
.iter()
|
|
.find(|i| i.item.url == watch(id))
|
|
.expect("collaboration item");
|
|
assert_eq!(item.status, LiveStatus::Past, "{id}");
|
|
assert_eq!(item.item.view_count, views, "{id}");
|
|
assert_eq!(item.item.upload_date_relative, age, "{id}");
|
|
assert_eq!(item.item.duration_seconds, duration, "{id}");
|
|
assert_eq!(item.item.uploader_name, "", "{id}");
|
|
}
|
|
}
|
|
|
|
/// The NASA first page with one cell changed by `edit`.
|
|
fn nasa_with_cell(index: usize, edit: impl FnOnce(&mut Value)) -> LiveTabPage {
|
|
let mut body = json(NASA_LIVE);
|
|
let tabs = body["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
|
|
.as_array_mut()
|
|
.expect("tabs");
|
|
let selected = tabs
|
|
.iter_mut()
|
|
.find(|t| t["tabRenderer"]["selected"] == Value::Bool(true))
|
|
.expect("selected tab");
|
|
edit(&mut selected["tabRenderer"]["content"]["richGridRenderer"]["contents"][index]);
|
|
parse_live_tab(&body, &en_gb())
|
|
}
|
|
|
|
#[test]
|
|
fn an_item_without_a_badge_is_listed_as_past_with_its_age_and_views() {
|
|
// No capture had one. Past keeps it playable; scheduled would not.
|
|
let p = nasa_with_cell(4, |cell| {
|
|
cell["richItemRenderer"]["content"]["lockupViewModel"]["contentImage"]
|
|
["thumbnailViewModel"]["overlays"] = Value::Array(Vec::new());
|
|
});
|
|
let item = &p.items[4];
|
|
assert_eq!(item.item.url, watch("UleiRdWFG7o"));
|
|
assert_eq!(item.status, LiveStatus::Past);
|
|
assert_eq!(item.item.duration_seconds, 0);
|
|
assert_eq!(item.item.view_count, 75_000);
|
|
assert_eq!(item.item.upload_date_relative, "Streamed 2 days ago");
|
|
}
|
|
|
|
#[test]
|
|
fn a_badge_with_digits_that_is_not_a_clock_is_listed_as_past() {
|
|
// Not seen in any capture. Only a badge with no digit reads as
|
|
// scheduled, so an unknown duration shape is not taken for one.
|
|
let p = nasa_with_cell(4, |cell| {
|
|
cell["richItemRenderer"]["content"]["lockupViewModel"]["contentImage"]
|
|
["thumbnailViewModel"]["overlays"][0]["thumbnailBottomOverlayViewModel"]["badges"][0]
|
|
["thumbnailBadgeViewModel"]["text"] = Value::String("28:1".into());
|
|
});
|
|
let item = &p.items[4];
|
|
assert_eq!(item.item.url, watch("UleiRdWFG7o"));
|
|
assert_eq!(item.status, LiveStatus::Past);
|
|
assert_eq!(item.item.duration_seconds, 0);
|
|
assert_eq!(item.item.view_count, 75_000);
|
|
assert_eq!(item.item.upload_date_relative, "Streamed 2 days ago");
|
|
}
|
|
|
|
#[test]
|
|
fn a_badge_that_is_a_word_reads_as_scheduled() {
|
|
// The past stream's badge replaced by the Finnish scheduled badge.
|
|
let p = nasa_with_cell(4, |cell| {
|
|
cell["richItemRenderer"]["content"]["lockupViewModel"]["contentImage"]
|
|
["thumbnailViewModel"]["overlays"][0]["thumbnailBottomOverlayViewModel"]["badges"][0]
|
|
["thumbnailBadgeViewModel"]["text"] = Value::String("Tulossa".into());
|
|
});
|
|
assert_eq!(
|
|
p.items[4].status,
|
|
LiveStatus::Scheduled {
|
|
waiting: Some("75k views".into()),
|
|
start_text: Some("Streamed 2 days ago".into()),
|
|
start_time: None,
|
|
}
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_bare_lockup_cell_is_read_like_a_wrapped_one() {
|
|
// The Videos tab parser accepts a lockup without its richItemRenderer
|
|
// wrapper; no Live tab capture had one.
|
|
let p = nasa_with_cell(2, |cell| {
|
|
let lockup = cell["richItemRenderer"]["content"]["lockupViewModel"].take();
|
|
*cell = serde_json::json!({ "lockupViewModel": lockup });
|
|
});
|
|
assert_eq!(kinds(&p), run(2, 2, 26));
|
|
assert_eq!(p.items[2].item.url, watch("ROboUHfPUrU"));
|
|
}
|
|
|
|
#[test]
|
|
fn a_channel_without_a_live_tab_gives_an_empty_page() {
|
|
// Measured: the Live params on such a channel select its Home tab.
|
|
let p = page(KURZGESAGT_LIVE_REQUEST, &en_gb());
|
|
assert!(p.items.is_empty());
|
|
assert_eq!(p.continuation, None);
|
|
}
|
|
|
|
#[test]
|
|
fn another_selected_grid_tab_is_not_read_as_the_live_tab() {
|
|
// The NASA page with its selected tab's URL pointed at the Videos tab:
|
|
// the same grid of items, which must not come back as streams.
|
|
let mut body = json(NASA_LIVE);
|
|
let tabs = body["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
|
|
.as_array_mut()
|
|
.expect("tabs");
|
|
let selected = tabs
|
|
.iter_mut()
|
|
.find(|t| t["tabRenderer"]["selected"] == Value::Bool(true))
|
|
.expect("selected tab");
|
|
selected["tabRenderer"]["endpoint"]["commandMetadata"]["webCommandMetadata"]["url"] =
|
|
Value::String("/@NASA/videos".into());
|
|
let p = parse_live_tab(&body, &en_gb());
|
|
assert!(p.items.is_empty());
|
|
assert_eq!(p.continuation, None);
|
|
}
|
|
|
|
#[test]
|
|
fn has_live_tab_reads_the_tab_list() {
|
|
assert!(has_live_tab(&json(NASA_HOME_TABS)));
|
|
assert!(has_live_tab(&json(NASA_LIVE)));
|
|
assert!(!has_live_tab(&json(KURZGESAGT_LIVE_REQUEST)));
|
|
assert!(!has_live_tab(&json("{}")));
|
|
}
|
|
|
|
/// Every JSON file under tests/fixtures/live_tab and
|
|
/// tests/fixtures/feed_lookup, read from disk so that a fixture added
|
|
/// later is covered without being listed here.
|
|
fn every_fixture() -> Vec<(String, String)> {
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
|
|
let mut out = Vec::new();
|
|
for dir in ["live_tab", "feed_lookup"] {
|
|
for entry in std::fs::read_dir(root.join(dir)).expect("fixture directory") {
|
|
let path = entry.expect("directory entry").path();
|
|
if path.extension().is_some_and(|e| e == "json") {
|
|
let file = path.file_name().expect("file name").to_string_lossy();
|
|
let text = std::fs::read_to_string(&path).expect("fixture text");
|
|
out.push((format!("{dir}/{file}"), text));
|
|
}
|
|
}
|
|
}
|
|
out.sort();
|
|
out
|
|
}
|
|
|
|
/// The first dotted IPv4 address in `text`: four groups of one to three
|
|
/// digits, each at most 255, standing alone rather than inside a longer
|
|
/// run of digits and dots ("20.772.51.34" is a duration, not an address).
|
|
fn first_ipv4(text: &str) -> Option<&str> {
|
|
text.split(|c: char| !(c.is_ascii_digit() || c == '.'))
|
|
.find(|run| {
|
|
let groups: Vec<&str> = run.split('.').collect();
|
|
groups.len() == 4
|
|
&& groups
|
|
.iter()
|
|
.all(|g| (1..=3).contains(&g.len()) && g.parse::<u16>().is_ok_and(|n| n <= 255))
|
|
})
|
|
}
|
|
|
|
#[test]
|
|
fn first_ipv4_finds_an_address_and_nothing_else() {
|
|
assert_eq!(first_ipv4(r#"{"a":"203.0.113.7"}"#), Some("203.0.113.7"));
|
|
assert_eq!(first_ipv4("ip%3D198.51.100.255&x"), Some("198.51.100.255"));
|
|
for text in [
|
|
"20.772.51.34",
|
|
"1.42.26",
|
|
"2.20261020.01.00",
|
|
"256.1.1.1",
|
|
"1.2.3.4.5",
|
|
"1.2.3",
|
|
"hq720.jpg",
|
|
] {
|
|
assert_eq!(first_ipv4(text), None, "{text}");
|
|
}
|
|
}
|
|
|
|
/// What in a fixture's text marks it as carrying request-bound data, if
|
|
/// anything: a media host, visitor data, a tracking field, an IP parameter
|
|
/// (plain or URL-encoded), a bare IPv4 address, or a thumbnail URL with its
|
|
/// query string.
|
|
fn request_bound_data(text: &str) -> Option<String> {
|
|
for marker in [
|
|
"googlevideo",
|
|
"visitorData",
|
|
"trackingParams",
|
|
"TrackingParams",
|
|
"ip=",
|
|
"ip%3D",
|
|
"ip%3d",
|
|
] {
|
|
if text.contains(marker) {
|
|
return Some(marker.to_string());
|
|
}
|
|
}
|
|
if let Some(address) = first_ipv4(text) {
|
|
return Some(format!("IPv4 {address}"));
|
|
}
|
|
let parsed: Value = serde_json::from_str(text).ok()?;
|
|
let mut strings = Vec::new();
|
|
collect_strings(&parsed, &mut strings);
|
|
strings
|
|
.into_iter()
|
|
.find(|s| s.contains(".ytimg.com/") && s.contains('?'))
|
|
.map(|s| format!("thumbnail query {s}"))
|
|
}
|
|
|
|
#[test]
|
|
fn request_bound_data_is_found_when_planted() {
|
|
// Each entry is a clean NASA cell with one thing planted in it.
|
|
let clean = r#"{"contentId":"UleiRdWFG7o","url":"https://i.ytimg.com/vi/UleiRdWFG7o/hq720.jpg","badge":"20.772.51.34"}"#;
|
|
assert_eq!(request_bound_data(clean), None);
|
|
for (plant, found) in [
|
|
(
|
|
r#""x":"https://rr1---sn-abc.googlevideo.com/videoplayback""#,
|
|
"googlevideo",
|
|
),
|
|
(r#""visitorData":"CgtXYWJjZGVm""#, "visitorData"),
|
|
(r#""trackingParams":"CAAQ""#, "trackingParams"),
|
|
(r#""clickTrackingParams":"CAAQ""#, "TrackingParams"),
|
|
(r#""serviceTrackingParams":[]"#, "TrackingParams"),
|
|
(r#""u":"https://yt3.ggpht.com/a=s88?ip=203.0.113.7""#, "ip="),
|
|
(r#""u":"remote=ip%3D203.0.113.7""#, "ip%3D"),
|
|
(r#""u":"remote=ip%3d203.0.113.7""#, "ip%3d"),
|
|
(r#""remoteHost":"203.0.113.7""#, "IPv4 203.0.113.7"),
|
|
(
|
|
r#""y":"https://i.ytimg.com/vi/UleiRdWFG7o/hq720.jpg?sqp=x""#,
|
|
"thumbnail query https://i.ytimg.com/vi/UleiRdWFG7o/hq720.jpg?sqp=x",
|
|
),
|
|
] {
|
|
let planted = clean.replacen('{', &format!("{{{plant},"), 1);
|
|
assert_eq!(
|
|
request_bound_data(&planted).as_deref(),
|
|
Some(found),
|
|
"{plant}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn the_fixtures_carry_no_request_bound_data() {
|
|
let fixtures = every_fixture();
|
|
let names: Vec<&str> = fixtures.iter().map(|(name, _)| name.as_str()).collect();
|
|
// The walk reached both directories: a moved directory cannot pass
|
|
// by finding nothing to check.
|
|
for expected in [
|
|
"live_tab/nasa_live.json",
|
|
"live_tab/nasa_live_da.json",
|
|
"feed_lookup/live_now.json",
|
|
"feed_lookup/past_stream.json",
|
|
"feed_lookup/scheduled.json",
|
|
] {
|
|
assert!(names.contains(&expected), "{expected} not in {names:?}");
|
|
}
|
|
for (name, text) in &fixtures {
|
|
json(text);
|
|
assert_eq!(request_bound_data(text), None, "{name}");
|
|
}
|
|
}
|
|
|
|
fn collect_strings<'a>(v: &'a Value, out: &mut Vec<&'a str>) {
|
|
match v {
|
|
Value::String(s) => out.push(s),
|
|
Value::Array(a) => a.iter().for_each(|x| collect_strings(x, out)),
|
|
Value::Object(m) => m.values().for_each(|x| collect_strings(x, out)),
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
/// Serves `first` to a browse request and `next` to a continuation request,
|
|
/// and records every request it is sent.
|
|
struct FixtureDownloader {
|
|
first: String,
|
|
next: String,
|
|
seen: Mutex<Vec<(String, Value)>>,
|
|
}
|
|
|
|
impl FixtureDownloader {
|
|
fn new(first: &str, next: &str) -> Arc<Self> {
|
|
Arc::new(FixtureDownloader {
|
|
first: first.to_string(),
|
|
next: next.to_string(),
|
|
seen: Mutex::new(Vec::new()),
|
|
})
|
|
}
|
|
}
|
|
|
|
impl Downloader for FixtureDownloader {
|
|
fn execute(&self, request: Request) -> Result<Response, NetworkError> {
|
|
let body: Value =
|
|
serde_json::from_slice(request.body().unwrap_or(b"{}")).unwrap_or(Value::Null);
|
|
let reply = if body.get("continuation").is_some() {
|
|
self.next.clone()
|
|
} else {
|
|
self.first.clone()
|
|
};
|
|
self.seen
|
|
.lock()
|
|
.unwrap()
|
|
.push((request.url().to_string(), body));
|
|
Ok(Response::new(
|
|
200,
|
|
"OK",
|
|
Default::default(),
|
|
reply,
|
|
request.url(),
|
|
))
|
|
}
|
|
}
|
|
|
|
/// The fetch tests install a downloader and a localization in the
|
|
/// process-wide NewPipe, so they take turns.
|
|
static NEWPIPE: Mutex<()> = Mutex::new(());
|
|
|
|
fn install(downloader: &Arc<FixtureDownloader>, localization: Localization, country: &str) {
|
|
NewPipe::init_full(
|
|
downloader.clone(),
|
|
localization,
|
|
ContentCountry::new(country),
|
|
);
|
|
}
|
|
|
|
fn start_time(item: &LiveTabItem) -> Option<i64> {
|
|
match &item.status {
|
|
LiveStatus::Scheduled { start_time, .. } => *start_time,
|
|
other => panic!("expected scheduled, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
/// The NASA first page's first scheduled stream as the one item of a
|
|
/// continuation response.
|
|
fn continuation_with_a_scheduled_stream() -> String {
|
|
let body = json(NASA_LIVE);
|
|
let tabs = body["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
|
|
.as_array()
|
|
.expect("tabs");
|
|
let selected = tabs
|
|
.iter()
|
|
.find(|t| t["tabRenderer"]["selected"] == Value::Bool(true))
|
|
.expect("selected tab");
|
|
let cell = selected["tabRenderer"]["content"]["richGridRenderer"]["contents"][0].clone();
|
|
serde_json::json!({
|
|
"onResponseReceivedActions": [
|
|
{ "appendContinuationItemsAction": { "continuationItems": [cell] } }
|
|
]
|
|
})
|
|
.to_string()
|
|
}
|
|
|
|
#[test]
|
|
fn fetching_a_page_costs_one_browse_post_and_asks_for_utc() {
|
|
let _turn = NEWPIPE.lock().unwrap_or_else(|e| e.into_inner());
|
|
let downloader = FixtureDownloader::new(NASA_LIVE, NASA_LIVE_PAGE2);
|
|
install(&downloader, en_gb(), "GB");
|
|
|
|
let first = channel_live(NASA_ID).expect("first page");
|
|
assert_eq!(kinds(&first), run(2, 2, 26));
|
|
let token = first.continuation.clone().expect("token");
|
|
let second = channel_live_continuation(&token).expect("second page");
|
|
assert_eq!(kinds(&second), run(0, 0, 30));
|
|
|
|
let seen = downloader.seen.lock().unwrap();
|
|
assert_eq!(seen.len(), 2, "one request per page");
|
|
let (url, body) = &seen[0];
|
|
assert_eq!(
|
|
url,
|
|
"https://www.youtube.com/youtubei/v1/browse?prettyPrint=false"
|
|
);
|
|
assert_eq!(body["browseId"], NASA_ID);
|
|
assert_eq!(body["params"], LIVE_PARAMS);
|
|
// start_time reads the start text as UTC. That holds only while the
|
|
// request asks for no offset and names no time zone, which YouTube
|
|
// applies over the offset (see LiveStatus::Scheduled).
|
|
let client = &body["context"]["client"];
|
|
assert_eq!(client["utcOffsetMinutes"], 0);
|
|
assert!(client.get("timeZone").is_none(), "{client}");
|
|
assert_eq!(client["hl"], "en-GB");
|
|
let (url, body) = &seen[1];
|
|
assert_eq!(
|
|
url,
|
|
"https://www.youtube.com/youtubei/v1/browse?prettyPrint=false"
|
|
);
|
|
assert_eq!(body["continuation"], Value::String(token));
|
|
assert!(body.get("browseId").is_none());
|
|
assert!(body.get("params").is_none());
|
|
assert!(body["context"]["client"].get("timeZone").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn both_fetches_read_the_start_with_the_localization_they_sent() {
|
|
let _turn = NEWPIPE.lock().unwrap_or_else(|e| e.into_inner());
|
|
let downloader = FixtureDownloader::new(NASA_LIVE, &continuation_with_a_scheduled_stream());
|
|
|
|
install(&downloader, en_gb(), "GB");
|
|
let first = channel_live(NASA_ID).expect("first page");
|
|
assert_eq!(start_time(&first.items[0]), Some(1_793_270_700));
|
|
let next = channel_live_continuation("token").expect("next page");
|
|
assert_eq!(kinds(&next), "S");
|
|
assert_eq!(start_time(&next.items[0]), Some(1_793_270_700));
|
|
|
|
// The same en-GB text fetched under en-US is not read as a time.
|
|
install(
|
|
&downloader,
|
|
Localization::new("en", Some("US".into())),
|
|
"US",
|
|
);
|
|
let first = channel_live(NASA_ID).expect("first page");
|
|
assert_eq!(start_time(&first.items[0]), None);
|
|
let next = channel_live_continuation("token").expect("next page");
|
|
assert_eq!(start_time(&next.items[0]), None);
|
|
|
|
let seen = downloader.seen.lock().unwrap();
|
|
let languages: Vec<&Value> = seen
|
|
.iter()
|
|
.map(|(_, body)| &body["context"]["client"]["hl"])
|
|
.collect();
|
|
assert_eq!(languages, ["en-GB", "en-GB", "en-US", "en-US"]);
|
|
}
|