New: Added Malayalam and Ukrainian languages

Closes #4662
Closes #5000

src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs

@@ -60,15 +60,15 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Title.S01.720p.TRUEFRENCH.WEB-DL.AAC2.0.H.264-BTN")]
         public void should_parse_language_french(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.French.Id);
+            result.Id.Should().Be(Language.French.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Spanish.HDTV.XviD-LOL")]
         public void should_parse_language_spanish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Spanish.Id);
+            result.Id.Should().Be(Language.Spanish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.German.HDTV.XviD-LOL")]
@@ -77,45 +77,45 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Series.Title.S01E03.Ger.Dub.AAC.1080p.WebDL.x264-TKP21")]
         public void should_parse_language_german(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.German.Id);
+            result.Id.Should().Be(Language.German.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Italian.HDTV.XviD-LOL")]
         [TestCase("Title.the.Series.1x19.ita.720p.bdmux.x264-novarip")]
         public void should_parse_language_italian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Italian.Id);
+            result.Id.Should().Be(Language.Italian.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Danish.HDTV.XviD-LOL")]
         public void should_parse_language_danish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Danish.Id);
+            result.Id.Should().Be(Language.Danish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Dutch.HDTV.XviD-LOL")]
         public void should_parse_language_dutch(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Dutch.Id);
+            result.Id.Should().Be(Language.Dutch.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Japanese.HDTV.XviD-LOL")]
         public void should_parse_language_japanese(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Japanese.Id);
+            result.Id.Should().Be(Language.Japanese.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Icelandic.HDTV.XviD-LOL")]
         [TestCase("Title.the.Series.S01E03.1080p.WEB-DL.DD5.1.H.264-SbR Icelandic")]
         public void should_parse_language_icelandic(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Icelandic.Id);
+            result.Id.Should().Be(Language.Icelandic.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Chinese.HDTV.XviD-LOL")]
@@ -133,23 +133,23 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("[喵萌奶茶屋&LoliHouse] 拳愿阿修罗 / Kengan Ashura - 17 [WebRip 1080p HEVC-10bit AAC][中日双语字幕]")]
         public void should_parse_language_chinese(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Chinese.Id);
+            result.Id.Should().Be(Language.Chinese.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Korean.HDTV.XviD-LOL")]
         public void should_parse_language_korean(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Korean.Id);
+            result.Id.Should().Be(Language.Korean.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Russian.HDTV.XviD-LOL")]
         [TestCase("Title.the.Series.S01E01.1080p.WEB-DL.Rus.Eng.TVKlondike")]
         public void should_parse_language_russian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Russian.Id);
+            result.Id.Should().Be(Language.Russian.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Polish.HDTV.XviD-LOL")]
@@ -164,64 +164,64 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Title.the.Series.2009.S01E14.DUB-PL.HDTV.XviD-LOL")]
         public void should_parse_language_polish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Polish.Id);
+            result.Id.Should().Be(Language.Polish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Vietnamese.HDTV.XviD-LOL")]
         public void should_parse_language_vietnamese(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Vietnamese.Id);
+            result.Id.Should().Be(Language.Vietnamese.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Swedish.HDTV.XviD-LOL")]
         public void should_parse_language_swedish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Swedish.Id);
+            result.Id.Should().Be(Language.Swedish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Norwegian.HDTV.XviD-LOL")]
         public void should_parse_language_norwegian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Norwegian.Id);
+            result.Id.Should().Be(Language.Norwegian.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Finnish.HDTV.XviD-LOL")]
         public void should_parse_language_finnish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Finnish.Id);
+            result.Id.Should().Be(Language.Finnish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Turkish.HDTV.XviD-LOL")]
         public void should_parse_language_turkish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Turkish.Id);
+            result.Id.Should().Be(Language.Turkish.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Portuguese.HDTV.XviD-LOL")]
         public void should_parse_language_portuguese(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Portuguese.Id);
+            result.Id.Should().Be(Language.Portuguese.Id);
         }
         
         [TestCase("Title.the.Series.S01E01.FLEMISH.HDTV.x264-BRiGAND")]
         public void should_parse_language_flemish(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Flemish.Id);
+            result.Id.Should().Be(Language.Flemish.Id);
         }
 
         [TestCase("Title.the.Series.S03E13.Greek.PDTV.XviD-Ouzo")]
         public void should_parse_language_greek(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Greek.Id);
+            result.Id.Should().Be(Language.Greek.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.HDTV.XviD.HUNDUB-LOL")]
@@ -229,44 +229,44 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Title.the.Series.2009.S01E14.HDTV.XviD.HUN-LOL")]
         public void should_parse_language_hungarian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Hungarian.Id);
+            result.Id.Should().Be(Language.Hungarian.Id);
         }
 
         [TestCase("Title.the.Series.S01-03.DVDRip.HebDub")]
         public void should_parse_language_hebrew(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Hebrew.Id);
+            result.Id.Should().Be(Language.Hebrew.Id);
         }
 
         [TestCase("Title.the.Series.S05E01.WEBRip.x264.AC3.LT.EN-CNN")]
         public void should_parse_language_lithuanian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Lithuanian.Id);
+            result.Id.Should().Be(Language.Lithuanian.Id);
         }
 
         [TestCase("Title.the.Series.​S07E11.​WEB Rip.​XviD.​Louige-​CZ.​EN.​5.​1")]
         public void should_parse_language_czech(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Czech.Id);
+            result.Id.Should().Be(Language.Czech.Id);
         }
 
         [TestCase("Series Title.S01.ARABIC.COMPLETE.720p.NF.WEBRip.x264-PTV")]
         public void should_parse_language_arabic(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Arabic.Id);  
+            result.Id.Should().Be(Language.Arabic.Id);  
         }
 
         [TestCase("The Shadow Series S01 E01-08 WebRip Dual Audio [Hindi 5.1 + English 5.1] 720p x264 AAC ESub")]
         [TestCase("The Final Sonarr (2020) S04 Complete 720p NF WEBRip [Hindi+English] Dual audio")]
         public void should_parse_language_hindi(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Hindi.Id);
+            result.Id.Should().Be(Language.Hindi.Id);
         }
 
         [TestCase("Title.the.Series.2009.S01E14.Bulgarian.HDTV.XviD-LOL")]
@@ -274,8 +274,26 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Title.the.Series.2009.S01E14.BG.AUDIO.HDTV.XviD-LOL")]
         public void should_parse_language_bulgarian(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Id.Should().Be(Language.Bulgarian.Id);
+            result.Id.Should().Be(Language.Bulgarian.Id);
+        }
+
+        [TestCase("Series Title S01E01 Malayalam.1080p.WebRip.AVC.5.1-Rjaa")]
+        [TestCase("Series Title S01E01 Malayalam DVDRip XviD 5.1 ESub MTR")]
+        [TestCase("Series.Title.S01E01.DVDRip.1CD.Malayalam.Xvid.MP3 @Mastitorrents")]
+        public void should_parse_language_malayalam(string postTitle)
+        {
+            var result = LanguageParser.ParseLanguage(postTitle);
+            result.Id.Should().Be(Language.Malayalam.Id);
+        }
+
+        [TestCase("Гало(Сезон 1, серії 1-5) / SeriesTitle(Season 1, episodes 1-5) (2022) WEBRip-AVC Ukr/Eng")]
+        [TestCase("Архів 81 (Сезон 1) / Series 81 (Season 1) (2022) WEB-DLRip-AVC Ukr/Eng | Sub Ukr/Eng")]
+        [TestCase("Книга Боби Фетта(Сезон 1) / Series Title(Season 1) (2021) WEB-DLRip Ukr/Eng")]
+        public void should_parse_language_ukrainian(string postTitle)
+        {
+            var result = LanguageParser.ParseLanguage(postTitle);
+            result.Id.Should().Be(Language.Ukrainian.Id);
         }
 
         [TestCase("Title.the.Russian.Series.S01E07.Cold.Action.HDTV.XviD-Droned")]
@@ -285,8 +303,8 @@ namespace NzbDrone.Core.Test.ParserTests
         [TestCase("Title The Spanish S02E02 Flodden 720p AMZN WEB-DL DDP5 1 H 264-NTb")]
         public void should_not_parse_series_or_episode_title(string postTitle)
         {
-            var result = Parser.Parser.ParseTitle(postTitle);
+            var result = LanguageParser.ParseLanguage(postTitle);
-            result.Language.Name.Should().Be(Language.English.Name);
+            result.Name.Should().Be(Language.English.Name);
         }
     }
 }

src/NzbDrone.Core/Datastore/Migration/169_add_malayalam _and_ukrainian_languages.cs

@@ -0,0 +1,125 @@
+using System.Collections.Generic;
+using System.Data;
+using System.Linq;
+using FluentMigrator;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Datastore.Migration.Framework;
+using NzbDrone.Core.Languages;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(169)]
+    public class add_malayalam_and_ukrainian_languages : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Execute.WithConnection(ConvertProfile);
+        }
+
+        private void ConvertProfile(IDbConnection conn, IDbTransaction tran)
+        {
+            var updater = new LanguageProfileUpdater169(conn, tran);
+
+            updater.AppendMissing();
+
+            updater.Commit();
+        }
+    }
+
+    public class LanguageProfile169 : ModelBase
+    {
+        public string Name { get; set; }
+        public List<LanguageProfileItem169> Languages { get; set; }
+        public bool UpgradeAllowed { get; set; }
+        public Language Cutoff { get; set; }
+    }
+
+    public class LanguageProfileItem169
+    {
+        public int Language { get; set; }
+        public bool Allowed { get; set; }
+    }
+
+
+    public class LanguageProfileUpdater169
+    {
+        private readonly IDbConnection _connection;
+        private readonly IDbTransaction _transaction;
+
+        private List<LanguageProfile169> _profiles;
+        private HashSet<LanguageProfile169> _changedProfiles = new HashSet<LanguageProfile169>();
+
+        public LanguageProfileUpdater169(IDbConnection conn, IDbTransaction tran)
+        {
+            _connection = conn;
+            _transaction = tran;
+
+            _profiles = GetProfiles();
+        }
+
+        public void Commit()
+        {
+            foreach (var profile in _changedProfiles)
+            {
+                using (var updateProfileCmd = _connection.CreateCommand())
+                {
+                    updateProfileCmd.Transaction = _transaction;
+                    updateProfileCmd.CommandText = "UPDATE LanguageProfiles SET Languages = ? WHERE Id = ?";
+                    updateProfileCmd.AddParameter(profile.Languages.ToJson());
+                    updateProfileCmd.AddParameter(profile.Id);
+
+                    updateProfileCmd.ExecuteNonQuery();
+                }
+            }
+
+            _changedProfiles.Clear();
+        }
+
+        public void AppendMissing()
+        {
+            foreach (var profile in _profiles)
+            {
+                var hash = new HashSet<int>(profile.Languages.Select(v => v.Language));
+
+                var missing = Language.All.Where(l => !hash.Contains(l.Id))
+                                          .OrderByDescending(l => l.Name)
+                                          .ToList();
+
+                if (missing.Any())
+                {
+                    profile.Languages.InsertRange(0, missing.Select(l => new LanguageProfileItem169 { Language = l.Id, Allowed = false }));
+
+                    _changedProfiles.Add(profile);
+                }
+            }
+        }
+
+        private List<LanguageProfile169> GetProfiles()
+        {
+            var profiles = new List<LanguageProfile169>();
+
+            using (var getProfilesCmd = _connection.CreateCommand())
+            {
+                getProfilesCmd.Transaction = _transaction;
+                getProfilesCmd.CommandText = @"SELECT Id, Name, Languages, UpgradeAllowed, Cutoff FROM LanguageProfiles";
+
+                using (var profileReader = getProfilesCmd.ExecuteReader())
+                {
+                    while (profileReader.Read())
+                    {
+                        profiles.Add(new LanguageProfile169
+                        {
+                            Id = profileReader.GetInt32(0),
+                            Name = profileReader.GetString(1),
+                            Languages = Json.Deserialize<List<LanguageProfileItem169>>(profileReader.GetString(2)),
+                            UpgradeAllowed = profileReader.GetBoolean(3),
+                            Cutoff = Language.FindById(profileReader.GetInt32(4))
+                        });
+                    }
+                }
+            }
+
+            return profiles;
+        }
+    }
+}

src/NzbDrone.Core/Languages/Language.cs

@@ -84,6 +84,8 @@ namespace NzbDrone.Core.Languages
         public static Language Arabic       { get { return new Language(26, "Arabic"); } }
         public static Language Hindi        { get { return new Language(27, "Hindi"); } }
         public static Language Bulgarian    { get { return new Language(28, "Bulgarian"); } }
+        public static Language Malayalam    { get { return new Language(29, "Malayalam"); } }
+        public static Language Ukrainian    { get { return new Language(30, "Ukrainian"); } }
 
 
         public static List<Language> All
@@ -120,7 +122,9 @@ namespace NzbDrone.Core.Languages
                     Czech,
                     Arabic,
                     Hindi,
-                    Bulgarian
+                    Bulgarian,
+                    Malayalam,
+                    Ukrainian
                 };
             }
         }

src/NzbDrone.Core/Parser/IsoLanguages.cs

@@ -36,7 +36,9 @@ namespace NzbDrone.Core.Parser
                                                                new IsoLanguage("cs", "ces", Language.Czech),
                                                                new IsoLanguage("ar", "ara", Language.Arabic),
                                                                new IsoLanguage("hi", "hin", Language.Hindi),
-                                                               new IsoLanguage("bg", "bul", Language.Bulgarian)
+                                                               new IsoLanguage("bg", "bul", Language.Bulgarian),
+                                                               new IsoLanguage("ml", "mal", Language.Malayalam),
+                                                               new IsoLanguage("uk", "ukr", Language.Ukrainian),
                                                            };
 
         public static IsoLanguage Find(string isoCode)

src/NzbDrone.Core/Parser/LanguageParser.cs

@@ -17,7 +17,7 @@ namespace NzbDrone.Core.Parser
                 new RegexReplace(@".*?[_. ](S\d{2}(?:E\d{2,4})*[_. ].*)", "$1", RegexOptions.Compiled | RegexOptions.IgnoreCase)
             };
 
-        private static readonly Regex LanguageRegex = new Regex(@"(?:\W|_)(?<italian>\b(?:ita|italian)\b)|(?<german>german\b|videomann|ger[. ]dub)|(?<flemish>flemish)|(?<greek>greek)|(?<french>(?:\W|_)(?:FR|VF|VF2|VFF|VFQ|TRUEFRENCH)(?:\W|_))|(?<russian>\brus\b)|(?<hungarian>\b(?:HUNDUB|HUN)\b)|(?<hebrew>\bHebDub\b)|(?<polish>\b(?:PL\W?DUB|DUB\W?PL|LEK\W?PL|PL\W?LEK)\b)|(?<chinese>\[(?:CH[ST]|BIG5|GB)\]|简|繁|字幕)|(?<bulgarian>\bbgaudio\b)|(?<spanish>\b(?:español|castellano)\b)",
+        private static readonly Regex LanguageRegex = new Regex(@"(?:\W|_)(?<italian>\b(?:ita|italian)\b)|(?<german>german\b|videomann|ger[. ]dub)|(?<flemish>flemish)|(?<greek>greek)|(?<french>(?:\W|_)(?:FR|VF|VF2|VFF|VFQ|TRUEFRENCH)(?:\W|_))|(?<russian>\brus\b)|(?<hungarian>\b(?:HUNDUB|HUN)\b)|(?<hebrew>\bHebDub\b)|(?<polish>\b(?:PL\W?DUB|DUB\W?PL|LEK\W?PL|PL\W?LEK)\b)|(?<chinese>\[(?:CH[ST]|BIG5|GB)\]|简|繁|字幕)|(?<bulgarian>\bbgaudio\b)|(?<spanish>\b(?:español|castellano)\b)|(?<ukrainian>\b(?:ukr)\b)",
                                                                 RegexOptions.IgnoreCase | RegexOptions.Compiled);
 
         private static readonly Regex CaseSensitiveLanguageRegex = new Regex(@"(?:(?i)(?<!SUB[\W|_|^]))(?:(?<lithuanian>\bLT\b)|(?<czech>\bCZ\b)|(?<polish>\bPL\b)|(?<bulgarian>\bBG\b))(?:(?i)(?![\W|_|^]SUB))",
@@ -96,6 +96,12 @@ namespace NzbDrone.Core.Parser
             if (lowerTitle.Contains("hindi"))
                 return Language.Hindi;
 
+            if (lowerTitle.Contains("malayalam"))
+                return Language.Malayalam;
+
+            if (lowerTitle.Contains("ukrainian"))
+                return Language.Ukrainian;
+
             if (lowerTitle.Contains("bulgarian"))
                 return Language.Bulgarian;
 
@@ -203,6 +209,9 @@ namespace NzbDrone.Core.Parser
             if (match.Groups["bulgarian"].Success)
                 return Language.Bulgarian;
 
+            if (match.Groups["ukrainian"].Success)
+                return Language.Ukrainian;
+
             return Language.Unknown;
         }
     }