From 1678f06870de39f6335d4e53ab2881747fc82cd1 Mon Sep 17 00:00:00 2001 From: Jendrik Weise Date: Sat, 25 Nov 2023 13:01:43 +0100 Subject: [PATCH] Clean up migration --- ...les_from_existing_subtitle_filesFixture.cs | 1 + .../AggregateSubtitleInfoFixture.cs | 4 +- ...rse_titles_from_existing_subtitle_files.cs | 56 +++++++++++++------ .../Aggregators/AggregateSubtitleInfo.cs | 16 ++++-- src/NzbDrone.Core/Parser/LanguageParser.cs | 27 ++++++++- .../Parser/Model/SubtitleTitleInfo.cs | 23 +------- 6 files changed, 80 insertions(+), 47 deletions(-) diff --git a/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs b/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs index 332a2f165..754c2308c 100644 --- a/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs +++ b/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs @@ -62,6 +62,7 @@ namespace NzbDrone.Core.Test.Datastore.Migration Id = 1, SeriesId = 1, RelativePath = episodePath, + OriginalFilePath = string.Empty, Quality = new { }.ToJson(), Size = 0, DateAdded = now, diff --git a/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs index 28c563dd8..d5e4a472a 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs @@ -23,7 +23,7 @@ namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Aggregation.Aggregators OriginalFilePath = originalFilePath }; - var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, path); + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path); subtitleTitleInfo.Title.Should().BeNull(); subtitleTitleInfo.Copy.Should().Be(0); @@ -40,7 +40,7 @@ namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Aggregation.Aggregators RelativePath = relativePath }; - var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, path); + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path); subtitleTitleInfo.LanguageTags.Should().NotContain("default"); } diff --git a/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs b/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs index e7d2020c4..5b8d18199 100644 --- a/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs +++ b/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs @@ -1,18 +1,24 @@ +using System; using System.Collections.Generic; using System.Data; -using System.Text.Json; -using System.Text.Json.Serialization; +using System.IO; +using System.Linq; using Dapper; using FluentMigrator; +using NLog; +using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Datastore.Migration.Framework; -using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Datastore.Migration { [Migration(198)] public class parse_title_from_existing_subtitle_files : NzbDroneMigrationBase { + private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo)); + protected override void MainDbUpgrade() { Alter.Table("SubtitleFiles").AddColumn("Title").AsString().Nullable(); @@ -27,18 +33,17 @@ namespace NzbDrone.Core.Datastore.Migration using (var cmd = conn.CreateCommand()) { cmd.Transaction = tran; - cmd.CommandText = "SELECT \"Id\", \"RelativePath\", \"EpisodeFileId\", \"Language\", \"LanguageTags\" FROM \"SubtitleFiles\""; + cmd.CommandText = "SELECT \"SubtitleFiles\".\"Id\", \"SubtitleFiles\".\"RelativePath\", \"EpisodeFiles\".\"RelativePath\", \"EpisodeFiles\".\"OriginalFilePath\" FROM \"SubtitleFiles\" JOIN \"EpisodeFiles\" ON \"SubtitleFiles\".\"EpisodeFileId\" = \"EpisodeFiles\".\"Id\""; using var reader = cmd.ExecuteReader(); while (reader.Read()) { var id = reader.GetInt32(0); var relativePath = reader.GetString(1); - var episodeFileId = reader.GetInt32(2); + var episodeFileRelativePath = reader.GetString(2); + var episodeFileOriginalFilePath = reader.GetString(3); - var episodeFile = conn.QuerySingle("SELECT * FROM \"EpisodeFiles\" WHERE \"Id\" = @Id", new { Id = episodeFileId }); - - var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, relativePath); + var subtitleTitleInfo = CleanSubtitleTitleInfo(episodeFileRelativePath, episodeFileOriginalFilePath, relativePath); updates.Add(new { @@ -51,18 +56,33 @@ namespace NzbDrone.Core.Datastore.Migration } } - var serializerSettings = new JsonSerializerOptions - { - AllowTrailingCommas = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNameCaseInsensitive = true, - DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true - }; - var updateSubtitleFilesSql = "UPDATE \"SubtitleFiles\" SET \"Title\" = @Title, \"Copy\" = @Copy, \"Language\" = @Language, \"LanguageTags\" = @LanguageTags, \"LastUpdated\" = CURRENT_TIMESTAMP WHERE \"Id\" = @Id"; conn.Execute(updateSubtitleFilesSql, updates, transaction: tran); } + + private static SubtitleTitleInfo CleanSubtitleTitleInfo(string relativePath, string originalFilePath, string path) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); + + var episodeFileTitle = Path.GetFileNameWithoutExtension(relativePath); + var originalEpisodeFileTitle = Path.GetFileNameWithoutExtension(originalFilePath) ?? string.Empty; + + if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) + { + Logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle); + + subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); + } + + var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !episodeFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) + { + Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + subtitleTitleInfo.LanguageTags = cleanedTags; + } + + return subtitleTitleInfo; + } } } diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs index eeb9222be..442b7db74 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs @@ -2,7 +2,6 @@ using System; using System.IO; using System.Linq; using NLog; -using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Download; using NzbDrone.Core.Extras.Subtitles; using NzbDrone.Core.Parser; @@ -12,13 +11,20 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateSubtitleInfo : IAggregateLocalEpisode { - private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo)); public int Order => 6; + private readonly Logger _logger; + + public AggregateSubtitleInfo(Logger logger) + { + _logger = logger; + } + public LocalEpisode Aggregate(LocalEpisode localEpisode, DownloadClientItem downloadClientItem) { var path = localEpisode.Path; var isSubtitleFile = SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(path)); + if (!isSubtitleFile) { return localEpisode; @@ -31,7 +37,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators return localEpisode; } - public static SubtitleTitleInfo CleanSubtitleTitleInfo(EpisodeFile episodeFile, string path) + public SubtitleTitleInfo CleanSubtitleTitleInfo(EpisodeFile episodeFile, string path) { var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); @@ -40,7 +46,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) { - Logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle); + _logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle); subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); } @@ -49,7 +55,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) { - Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + _logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); subtitleTitleInfo.LanguageTags = cleanedTags; } diff --git a/src/NzbDrone.Core/Parser/LanguageParser.cs b/src/NzbDrone.Core/Parser/LanguageParser.cs index 0e6caedca..c0ae05902 100644 --- a/src/NzbDrone.Core/Parser/LanguageParser.cs +++ b/src/NzbDrone.Core/Parser/LanguageParser.cs @@ -33,6 +33,8 @@ namespace NzbDrone.Core.Parser private static readonly Regex SubtitleLanguageTitleRegex = new Regex(".+?(\\.((?full|forced|foreign|default|cc|psdh|sdh)|(?[a-z]{2,3})))*\\.(?[^.]*)(\\.((?<tags2>full|forced|foreign|default|cc|psdh|sdh)|(?<iso_code>[a-z]{2,3})))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SubtitleTitleRegex = new Regex("((?<title>.+) - )?(?<copy>\\d+)", RegexOptions.Compiled); + public static List<Language> ParseLanguages(string title) { foreach (var regex in CleanSeriesTitleRegex) @@ -286,13 +288,36 @@ namespace NzbDrone.Core.Parser .Select(tag => tag.Value.ToLower()); var rawTitle = matchTitle.Groups["title"].Value; - return new SubtitleTitleInfo + var subtitleTitleInfo = new SubtitleTitleInfo { TitleFirst = matchTitle.Groups["tags1"].Captures.Empty(), LanguageTags = languageTags.ToList(), RawTitle = rawTitle, Language = language }; + + UpdateTitleAndCopyFromTitle(subtitleTitleInfo); + + return subtitleTitleInfo; + } + + public static void UpdateTitleAndCopyFromTitle(SubtitleTitleInfo subtitleTitleInfo) + { + if (subtitleTitleInfo.RawTitle is null) + { + subtitleTitleInfo.Title = null; + subtitleTitleInfo.Copy = 0; + } + else if (SubtitleTitleRegex.Match(subtitleTitleInfo.RawTitle) is var match && match.Success) + { + subtitleTitleInfo.Title = match.Groups["title"].Success ? match.Groups["title"].ToString() : null; + subtitleTitleInfo.Copy = int.Parse(match.Groups["copy"].ToString()); + } + else + { + subtitleTitleInfo.Title = subtitleTitleInfo.RawTitle; + subtitleTitleInfo.Copy = 0; + } } public static List<string> ParseLanguageTags(string fileName) diff --git a/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs index c757ceeb1..29ea84377 100644 --- a/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs @@ -1,34 +1,15 @@ -using System; using System.Collections.Generic; -using System.Text.RegularExpressions; using NzbDrone.Core.Languages; namespace NzbDrone.Core.Parser.Model { public class SubtitleTitleInfo { - private static readonly Regex SubtitleTitleRegex = new Regex("((?<title>.+) - )?(?<copy>\\d+)", RegexOptions.Compiled); public List<string> LanguageTags { get; set; } public Language Language { get; set; } public string RawTitle { get; set; } - private Lazy<(string Title, int Copy)> TitleAndCopy => new Lazy<(string title, int copy)>(() => - { - if (RawTitle is null) - { - return (null, 0); - } - - var match = SubtitleTitleRegex.Match(RawTitle); - - if (match.Success) - { - return (match.Groups["title"].Success ? match.Groups["title"].ToString() : null, int.Parse(match.Groups["copy"].ToString())); - } - - return (RawTitle, 0); - }); - public string Title => TitleAndCopy.Value.Title; - public int Copy => TitleAndCopy.Value.Copy; + public string Title { get; set; } + public int Copy { get; set; } public bool TitleFirst { get; set; } } }