fix date parsing

src/codegen/collect_playlist_dates.rs

@@ -84,15 +84,7 @@ async fn collect_dates() {
     serde_json::to_writer_pretty(file, &collected_dates).unwrap();
 }
 
-fn filter_str(string: &str) -> String {
+// #[test]
-    string
-        .to_lowercase()
-        .chars()
-        .filter(|c| c != &'\u{200b}' && !c.is_ascii_digit())
-        .collect()
-}
-
-#[test]
 fn write_samples_to_dict() {
     let json_path = Path::new("testfiles/date/playlist_samples.json").to_path_buf();
     let json_file = File::open(json_path).unwrap();
@@ -132,96 +124,17 @@ fn write_samples_to_dict() {
     ];
 
     for lang in langs {
-        let datestr_table = collected_dates.get(&lang).unwrap();
+        let mut datestr_tables = vec![collected_dates.get(&lang).unwrap()];
-        let mut month_words: HashMap<String, usize> = HashMap::new();
+        dict.get(&lang)
-        let mut num_order = "".to_owned();
+            .unwrap()
-
+            .equivalent
-        // Today/Yesterday
+            .iter()
-        let mut td_words: HashMap<String, i8> = HashMap::new();
+            .for_each(|l| datestr_tables.push(collected_dates.get(l).unwrap()));
-        {
-            let mut parse = |string: &str, n: i8| {
-                filter_str(string).split_whitespace().for_each(|word| {
-                    td_words
-                        .entry(word.to_owned())
-                        .and_modify(|e| *e = 0)
-                        .or_insert(n);
-                });
-            };
-
-            parse(datestr_table.get(&DateCase::Today).unwrap(), 1);
-            parse(datestr_table.get(&DateCase::Yesterday).unwrap(), 2);
-            parse(datestr_table.get(&DateCase::Ago).unwrap(), 0);
-            parse(datestr_table.get(&DateCase::Jan).unwrap(), 0);
-        }
-
-        // n days ago
-        {
-            let datestr = datestr_table.get(&DateCase::Ago).unwrap();
-            let tago = timeago::parse(lang, &datestr);
-            assert_eq!(
-                tago,
-                Some(TimeAgo {
-                    n: 3,
-                    unit: timeago::TimeUnit::Day
-                }),
-                "lang: {}, txt: {}",
-                lang,
-                datestr
-            );
-        }
-
-        // Absolute dates (Jan 3, 2020)
-        months.iter().enumerate().for_each(|(n, m)| {
-            let datestr = datestr_table.get(m).unwrap();
-
-            // Get order of numbers
-            let nums = util::parse_numeric_vec::<u32>(&datestr);
-            let date = dates[n];
-
-            let this_num_order = nums
-                .iter()
-                .map(|n| {
-                    if n == &date.0 {
-                        "Y"
-                    } else if n == &date.1 {
-                        "M"
-                    } else if n == &date.2 {
-                        "D"
-                    } else {
-                        panic!("invalid number {} in {}", n, datestr);
-                    }
-                })
-                .collect::<String>();
-
-            if num_order == "" {
-                num_order = this_num_order;
-            } else {
-                assert_eq!(this_num_order, num_order);
-            }
-
-            // Insert words into the map
-            filter_str(&datestr).split_whitespace().for_each(|word| {
-                month_words
-                    .entry(word.to_owned())
-                    .and_modify(|e| *e = 0)
-                    .or_insert(n + 1);
-            });
-        });
 
         let dict_entry = dict.entry(lang).or_default();
-        dict_entry.date_order = num_order;
+        let mut num_order = "".to_owned();
-        dict_entry.months = month_words
-            .iter()
-            .filter_map(|(word, m)| {
-                if *m == 0 {
-                    None
-                } else {
-                    Some((word.to_owned(), *m as u8))
-                }
-            })
-            .collect();
 
-        match lang {
+        let collect_nd_tokens = match lang {
             Language::Ja
             | Language::ZhCn
             | Language::ZhHk
@@ -232,25 +145,134 @@ fn write_samples_to_dict() {
             | Language::Ur
             | Language::Uz
             | Language::Te
+            | Language::PtPt
             // Singhalese YT translation is broken (today == tomorrow)
-            | Language::Si => {}
+            | Language::Si => false,
-            _ => {
+            _ => true,
-                dict_entry.timeago_nd_tokens = td_words
+        };
+
+        dict_entry.months = BTreeMap::new();
+
+        if collect_nd_tokens {
+            dict_entry.timeago_nd_tokens = BTreeMap::new();
+        }
+
+        for datestr_table in &datestr_tables {
+            let mut month_words: HashMap<String, usize> = HashMap::new();
+            let mut td_words: HashMap<String, i8> = HashMap::new();
+
+            // Today/Yesterday
+            {
+                let mut parse = |string: &str, n: i8| {
+                    timeago::filter_str(string)
+                        .split_whitespace()
+                        .for_each(|word| {
+                            td_words
+                                .entry(word.to_owned())
+                                .and_modify(|e| *e = 0)
+                                .or_insert(n);
+                        });
+                };
+
+                parse(datestr_table.get(&DateCase::Today).unwrap(), 1);
+                parse(datestr_table.get(&DateCase::Yesterday).unwrap(), 2);
+                parse(datestr_table.get(&DateCase::Ago).unwrap(), 0);
+                parse(datestr_table.get(&DateCase::Jan).unwrap(), 0);
+            }
+
+            // n days ago
+            {
+                let datestr = datestr_table.get(&DateCase::Ago).unwrap();
+                let tago = timeago::parse(lang, &datestr);
+                assert_eq!(
+                    tago,
+                    Some(TimeAgo {
+                        n: 3,
+                        unit: timeago::TimeUnit::Day
+                    }),
+                    "lang: {}, txt: {}",
+                    lang,
+                    datestr
+                );
+            }
+
+            // Absolute dates (Jan 3, 2020)
+            months.iter().enumerate().for_each(|(n, m)| {
+                let datestr = datestr_table.get(m).unwrap();
+
+                // Get order of numbers
+                let nums = util::parse_numeric_vec::<u32>(&datestr);
+                let date = dates[n];
+
+                let this_num_order = nums
                     .iter()
-                    .filter_map(|(word, n)| {
+                    .map(|n| {
-                        match n {
+                        if n == &date.0 {
-                            // Today
+                            "Y"
-                            1 => Some((word.to_owned(), "0D".to_owned())),
+                        } else if n == &date.1 {
-                            // Yesterday
+                            "M"
-                            2 => Some((word.to_owned(), "1D".to_owned())),
+                        } else if n == &date.2 {
-                            _ => None,
+                            "D"
+                        } else {
+                            panic!("invalid number {} in {}", n, datestr);
                         }
                     })
-                    .collect();
+                    .collect::<String>();
 
-                assert_eq!(dict_entry.timeago_nd_tokens.len(), 2, "lang: {}, nd_tokens: {:?}", lang, &dict_entry.timeago_nd_tokens);
+                if num_order == "" {
+                    num_order = this_num_order;
+                } else {
+                    assert_eq!(this_num_order, num_order, "lang: {}", lang);
+                }
+
+                // Insert words into the map
+                timeago::filter_str(&datestr)
+                    .split_whitespace()
+                    .for_each(|word| {
+                        month_words
+                            .entry(word.to_owned())
+                            .and_modify(|e| *e = 0)
+                            .or_insert(n + 1);
+                    });
+            });
+
+            month_words.iter().for_each(|(word, m)| {
+                if *m != 0 {
+                    dict_entry.months.insert(word.to_owned(), *m as u8);
+                };
+            });
+
+            if collect_nd_tokens {
+                td_words.iter().for_each(|(word, n)| {
+                    match n {
+                        // Today
+                        1 => {
+                            dict_entry
+                                .timeago_nd_tokens
+                                .insert(word.to_owned(), "0D".to_owned());
+                        }
+                        // Yesterday
+                        2 => {
+                            dict_entry
+                                .timeago_nd_tokens
+                                .insert(word.to_owned(), "1D".to_owned());
+                        }
+                        _ => {}
+                    };
+                });
+
+                if datestr_tables.len() == 1 {
+                    assert_eq!(
+                        dict_entry.timeago_nd_tokens.len(),
+                        2,
+                        "lang: {}, nd_tokens: {:?}",
+                        lang,
+                        &dict_entry.timeago_nd_tokens
+                    );
+                }
             }
         }
+        dict_entry.date_order = num_order;
     }
 
     super::write_dict(&dict);

src/codegen/gen_dictionary.rs

@@ -27,7 +27,7 @@ fn parse_tu(tu: &str) -> (u8, Option<TimeUnit>) {
     }
 }
 
-#[test]
+// #[test]
 fn generate_dictionary() {
     let dict = super::read_dict();
 

src/dictionary.rs

@@ -815,23 +815,24 @@ pub fn entry(lang: Language) -> Entry {
             months: ::phf::Map {
                 key: 12913932095322966823,
                 disps: &[
-                    (11, 10),
+                    (8, 0),
-                    (0, 0),
+                    (5, 8),
-                    (0, 0),
+                    (4, 0),
                 ],
                 entries: &[
-                    ("oct", 10),
-                    ("sep", 9),
-                    ("jul", 7),
-                    ("jun", 6),
-                    ("may", 5),
                     ("nov", 11),
+                    ("sept", 9),
                     ("apr", 4),
-                    ("mar", 3),
-                    ("jan", 1),
-                    ("aug", 8),
-                    ("feb", 2),
                     ("dec", 12),
+                    ("mar", 3),
+                    ("jun", 6),
+                    ("sep", 9),
+                    ("may", 5),
+                    ("jul", 7),
+                    ("jan", 1),
+                    ("oct", 10),
+                    ("feb", 2),
+                    ("aug", 8),
                 ],
             },
             timeago_nd_tokens: ::phf::Map {
@@ -1199,25 +1200,26 @@ pub fn entry(lang: Language) -> Entry {
             },
             date_order: &[DateCmp::D, DateCmp::Y],
             months: ::phf::Map {
-                key: 15467950696543387533,
+                key: 12913932095322966823,
                 disps: &[
-                    (0, 0),
+                    (9, 0),
-                    (5, 2),
+                    (7, 6),
-                    (2, 1),
+                    (1, 5),
                 ],
                 entries: &[
-                    ("déc.", 12),
-                    ("mai", 5),
-                    ("sept.", 9),
-                    ("nov.", 11),
-                    ("mars", 3),
-                    ("févr.", 2),
-                    ("avr.", 4),
-                    ("janv.", 1),
                     ("août", 8),
-                    ("oct.", 10),
+                    ("avr.", 4),
                     ("juil.", 7),
+                    ("févr.", 2),
+                    ("oct.", 10),
+                    ("déc.", 12),
+                    ("janv.", 1),
+                    ("mars", 3),
                     ("juin", 6),
+                    ("juill.", 7),
+                    ("nov.", 11),
+                    ("sept.", 9),
+                    ("mai", 5),
                 ],
             },
             timeago_nd_tokens: ::phf::Map {
@@ -2121,25 +2123,25 @@ pub fn entry(lang: Language) -> Entry {
             },
             date_order: &[DateCmp::Y, DateCmp::D],
             months: ::phf::Map {
-                key: 12913932095322966823,
+                key: 2980949210194914378,
                 disps: &[
                     (0, 0),
-                    (4, 0),
+                    (5, 0),
-                    (0, 10),
+                    (0, 0),
                 ],
                 entries: &[
-                    ("-янв.", 1),
+                    ("фев.", 2),
-                    ("-июн.", 6),
+                    ("ноя.", 11),
-                    ("-апр.", 4),
+                    ("авг.", 8),
-                    ("-мар.", 3),
+                    ("май", 5),
-                    ("-дек.", 12),
+                    ("янв.", 1),
-                    ("-май", 5),
+                    ("окт.", 10),
-                    ("-июл.", 7),
+                    ("июл.", 7),
-                    ("-авг.", 8),
+                    ("сен.", 9),
-                    ("-ноя.", 11),
+                    ("июн.", 6),
-                    ("-фев.", 2),
+                    ("мар.", 3),
-                    ("-сен.", 9),
+                    ("дек.", 12),
-                    ("-окт.", 10),
+                    ("апр.", 4),
                 ],
             },
             timeago_nd_tokens: ::phf::Map {
@@ -2961,7 +2963,7 @@ pub fn entry(lang: Language) -> Entry {
                 ],
             },
         },
-        Language::Pt | Language::PtPt => Entry {
+        Language::Pt => Entry {
             by_char: false,
             timeago_tokens: ::phf::Map {
                 key: 10121458955350035957,
@@ -3021,6 +3023,51 @@ pub fn entry(lang: Language) -> Entry {
                 ],
             },
         },
+        Language::PtPt => Entry {
+            by_char: false,
+            timeago_tokens: ::phf::Map {
+                key: 10121458955350035957,
+                disps: &[
+                    (6, 9),
+                    (0, 0),
+                    (2, 6),
+                ],
+                entries: &[
+                    ("segundos", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
+                    ("dia", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
+                    ("ano", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
+                    ("meses", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
+                    ("anos", TaToken { n: 1, unit: Some(TimeUnit::Year) }),
+                    ("minuto", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
+                    ("semana", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
+                    ("hora", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
+                    ("semanas", TaToken { n: 1, unit: Some(TimeUnit::Week) }),
+                    ("segundo", TaToken { n: 1, unit: Some(TimeUnit::Second) }),
+                    ("minutos", TaToken { n: 1, unit: Some(TimeUnit::Minute) }),
+                    ("horas", TaToken { n: 1, unit: Some(TimeUnit::Hour) }),
+                    ("mês", TaToken { n: 1, unit: Some(TimeUnit::Month) }),
+                    ("dias", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
+                ],
+            },
+            date_order: &[DateCmp::D, DateCmp::M, DateCmp::Y],
+            months: ::phf::Map {
+                key: 12913932095322966823,
+                disps: &[
+                ],
+                entries: &[
+                ],
+            },
+            timeago_nd_tokens: ::phf::Map {
+                key: 12913932095322966823,
+                disps: &[
+                    (1, 0),
+                ],
+                entries: &[
+                    ("hoje", TaToken { n: 0, unit: Some(TimeUnit::Day) }),
+                    ("ontem", TaToken { n: 1, unit: Some(TimeUnit::Day) }),
+                ],
+            },
+        },
         Language::Ro => Entry {
             by_char: false,
             timeago_tokens: ::phf::Map {
@@ -3950,25 +3997,25 @@ pub fn entry(lang: Language) -> Entry {
             },
             date_order: &[DateCmp::D, DateCmp::Y],
             months: ::phf::Map {
-                key: 10121458955350035957,
+                key: 12913932095322966823,
                 disps: &[
+                    (2, 5),
                     (0, 0),
-                    (5, 9),
+                    (9, 6),
-                    (0, 1),
                 ],
                 entries: &[
-                    ("-avg,", 8),
+                    ("mar,", 3),
-                    ("-iyn,", 6),
+                    ("may,", 5),
-                    ("-sen,", 9),
+                    ("noy,", 11),
-                    ("-noy,", 11),
+                    ("avg,", 8),
-                    ("-fev,", 2),
+                    ("iyl,", 7),
-                    ("-apr,", 4),
+                    ("iyn,", 6),
-                    ("-yan,", 1),
+                    ("yan,", 1),
-                    ("-may,", 5),
+                    ("fev,", 2),
-                    ("-mar,", 3),
+                    ("dek,", 12),
-                    ("-dek,", 12),
+                    ("okt,", 10),
-                    ("-okt,", 10),
+                    ("sen,", 9),
-                    ("-iyl,", 7),
+                    ("apr,", 4),
                 ],
             },
             timeago_nd_tokens: ::phf::Map {
@@ -4160,25 +4207,25 @@ pub fn entry(lang: Language) -> Entry {
             },
             date_order: &[DateCmp::D, DateCmp::Y],
             months: ::phf::Map {
-                key: 15467950696543387533,
+                key: 7485420634051515786,
                 disps: &[
-                    (6, 0),
+                    (2, 0),
-                    (0, 0),
+                    (1, 9),
-                    (2, 1),
+                    (1, 10),
                 ],
                 entries: &[
-                    ("ka-jun", 6),
+                    ("dis", 12),
-                    ("ka-eph", 4),
+                    ("jul", 7),
-                    ("ka-sep", 9),
+                    ("aga", 8),
-                    ("ka-dis", 12),
+                    ("okt", 10),
-                    ("ka-jan", 1),
+                    ("feb", 2),
-                    ("ka-nov", 11),
+                    ("nov", 11),
-                    ("ka-mey", 5),
+                    ("sep", 9),
-                    ("ka-mas", 3),
+                    ("mas", 3),
-                    ("ka-feb", 2),
+                    ("eph", 4),
-                    ("ka-okt", 10),
+                    ("mey", 5),
-                    ("ka-aga", 8),
+                    ("jan", 1),
-                    ("ka-jul", 7),
+                    ("jun", 6),
                 ],
             },
             timeago_nd_tokens: ::phf::Map {

src/timeago.rs

@@ -90,11 +90,19 @@ impl Mul<u8> for TimeAgo {
     }
 }
 
-fn filter_str(string: &str) -> String {
+pub fn filter_str(string: &str) -> String {
     string
         .to_lowercase()
         .chars()
-        .filter(|c| c != &'\u{200b}' && !c.is_ascii_digit())
+        .filter_map(|c| {
+            if c == '\u{200b}' || c.is_ascii_digit() {
+                None
+            } else if c == '-' {
+                Some(' ')
+            } else {
+                Some(c)
+            }
+        })
         .collect()
 }
 
@@ -439,6 +447,7 @@ mod tests {
     #[case(Language::En, "Updated today", Some(ParsedDate::Relative(TimeAgo { n: 0, unit: TimeUnit::Day })))]
     #[case(Language::En, "Updated yesterday", Some(ParsedDate::Relative(TimeAgo { n: 1, unit: TimeUnit::Day })))]
     #[case(Language::En, "Updated 2 days ago", Some(ParsedDate::Relative(TimeAgo { n: 2, unit: TimeUnit::Day })))]
+    #[case(Language::Si, "ඊයේ යාවත්කාලීන කරන ලදී", Some(ParsedDate::Relative(TimeAgo { n: 1, unit: TimeUnit::Day })))]
     #[case(
         Language::En,
         "Last updated on Jun 04, 2003",
@@ -473,7 +482,11 @@ mod tests {
             assert_eq!(
                 parse_date(*lang, samples.get("Yesterday").unwrap()),
                 Some(ParsedDate::Relative(TimeAgo {
-                    n: 1,
+                    // YT's Singhalese translation has an error (yesterday == today)
+                    n: match lang {
+                        Language::Si => 0,
+                        _ => 1,
+                    },
                     unit: TimeUnit::Day
                 })),
                 "lang: {}",

testfiles/date/dictionary.json

@@ -518,7 +518,8 @@
       "may": 5,
       "nov": 11,
       "oct": 10,
-      "sep": 9
+      "sep": 9,
+      "sept": 9
     },
     "timeago_nd_tokens": {
       "today": "0D",
@@ -761,6 +762,7 @@
       "févr.": 2,
       "janv.": 1,
       "juil.": 7,
+      "juill.": 7,
       "juin": 6,
       "mai": 5,
       "mars": 3,
@@ -1332,18 +1334,18 @@
     },
     "date_order": "YD",
     "months": {
-      "-авг.": 8,
+      "авг.": 8,
-      "-апр.": 4,
+      "апр.": 4,
-      "-дек.": 12,
+      "дек.": 12,
-      "-июл.": 7,
+      "июл.": 7,
-      "-июн.": 6,
+      "июн.": 6,
-      "-май": 5,
+      "май": 5,
-      "-мар.": 3,
+      "мар.": 3,
-      "-ноя.": 11,
+      "ноя.": 11,
-      "-окт.": 10,
+      "окт.": 10,
-      "-сен.": 9,
+      "сен.": 9,
-      "-фев.": 2,
+      "фев.": 2,
-      "-янв.": 1
+      "янв.": 1
     },
     "timeago_nd_tokens": {
       "бүгүн": "0D",
@@ -1855,9 +1857,7 @@
     }
   },
   "pt": {
-    "equivalent": [
+    "equivalent": [],
-      "pt-PT"
-    ],
     "by_char": false,
     "timeago_tokens": {
       "ano": "Y",
@@ -1895,6 +1895,32 @@
       "ontem": "1D"
     }
   },
+  "pt-PT": {
+    "equivalent": [],
+    "by_char": false,
+    "timeago_tokens": {
+      "ano": "Y",
+      "anos": "Y",
+      "dia": "D",
+      "dias": "D",
+      "hora": "h",
+      "horas": "h",
+      "meses": "M",
+      "minuto": "m",
+      "minutos": "m",
+      "mês": "M",
+      "segundo": "s",
+      "segundos": "s",
+      "semana": "W",
+      "semanas": "W"
+    },
+    "date_order": "DMY",
+    "months": {},
+    "timeago_nd_tokens": {
+      "hoje": "0D",
+      "ontem": "1D"
+    }
+  },
   "ro": {
     "equivalent": [],
     "by_char": false,
@@ -2487,18 +2513,18 @@
     },
     "date_order": "DY",
     "months": {
-      "-apr,": 4,
+      "apr,": 4,
-      "-avg,": 8,
+      "avg,": 8,
-      "-dek,": 12,
+      "dek,": 12,
-      "-fev,": 2,
+      "fev,": 2,
-      "-iyl,": 7,
+      "iyl,": 7,
-      "-iyn,": 6,
+      "iyn,": 6,
-      "-mar,": 3,
+      "mar,": 3,
-      "-may,": 5,
+      "may,": 5,
-      "-noy,": 11,
+      "noy,": 11,
-      "-okt,": 10,
+      "okt,": 10,
-      "-sen,": 9,
+      "sen,": 9,
-      "-yan,": 1
+      "yan,": 1
     },
     "timeago_nd_tokens": {
       "bugun": "0D",
@@ -2604,18 +2630,18 @@
     },
     "date_order": "DY",
     "months": {
-      "ka-aga": 8,
+      "aga": 8,
-      "ka-dis": 12,
+      "dis": 12,
-      "ka-eph": 4,
+      "eph": 4,
-      "ka-feb": 2,
+      "feb": 2,
-      "ka-jan": 1,
+      "jan": 1,
-      "ka-jul": 7,
+      "jul": 7,
-      "ka-jun": 6,
+      "jun": 6,
-      "ka-mas": 3,
+      "mas": 3,
-      "ka-mey": 5,
+      "mey": 5,
-      "ka-nov": 11,
+      "nov": 11,
-      "ka-okt": 10,
+      "okt": 10,
-      "ka-sep": 9
+      "sep": 9
     },
     "timeago_nd_tokens": {
       "izolo": "1D",