Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean up migration

This commit is contained in:
Jendrik Weise 2023-11-25 13:01:43 +01:00
parent 4cbd3c4b5a
commit 1678f06870
6 changed files with 80 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs
View File

@ -62,6 +62,7 @@ namespace NzbDrone.Core.Test.Datastore.Migration
Id = 1,
SeriesId = 1,
RelativePath = episodePath,
OriginalFilePath = string.Empty,
Quality = new { }.ToJson(),
Size = 0,
DateAdded = now,

4
src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs
View File

@ -23,7 +23,7 @@ namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Aggregation.Aggregators
OriginalFilePath = originalFilePath
};
var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, path);
var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path);
subtitleTitleInfo.Title.Should().BeNull();
subtitleTitleInfo.Copy.Should().Be(0);
@ -40,7 +40,7 @@ namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Aggregation.Aggregators
RelativePath = relativePath
};
var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, path);
var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path);
subtitleTitleInfo.LanguageTags.Should().NotContain("default");
}

56
src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs
View File

@ -1,18 +1,24 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.IO;
using System.Linq;
using Dapper;
using FluentMigrator;
using NLog;
using NzbDrone.Common.Instrumentation;
using NzbDrone.Core.Datastore.Migration.Framework;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(198)]
public class parse_title_from_existing_subtitle_files : NzbDroneMigrationBase
{
private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo));
protected override void MainDbUpgrade()
{
Alter.Table("SubtitleFiles").AddColumn("Title").AsString().Nullable();
@ -27,18 +33,17 @@ namespace NzbDrone.Core.Datastore.Migration
using (var cmd = conn.CreateCommand())
{
cmd.Transaction = tran;
cmd.CommandText = "SELECT \"Id\", \"RelativePath\", \"EpisodeFileId\", \"Language\", \"LanguageTags\" FROM \"SubtitleFiles\"";
cmd.CommandText = "SELECT \"SubtitleFiles\".\"Id\", \"SubtitleFiles\".\"RelativePath\", \"EpisodeFiles\".\"RelativePath\", \"EpisodeFiles\".\"OriginalFilePath\" FROM \"SubtitleFiles\" JOIN \"EpisodeFiles\" ON \"SubtitleFiles\".\"EpisodeFileId\" = \"EpisodeFiles\".\"Id\"";
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
var id = reader.GetInt32(0);
var relativePath = reader.GetString(1);
var episodeFileId = reader.GetInt32(2);
var episodeFileRelativePath = reader.GetString(2);
var episodeFileOriginalFilePath = reader.GetString(3);
var episodeFile = conn.QuerySingle<EpisodeFile>("SELECT * FROM \"EpisodeFiles\" WHERE \"Id\" = @Id", new { Id = episodeFileId });
var subtitleTitleInfo = AggregateSubtitleInfo.CleanSubtitleTitleInfo(episodeFile, relativePath);
var subtitleTitleInfo = CleanSubtitleTitleInfo(episodeFileRelativePath, episodeFileOriginalFilePath, relativePath);
updates.Add(new
{
@ -51,18 +56,33 @@ namespace NzbDrone.Core.Datastore.Migration
}
}
var serializerSettings = new JsonSerializerOptions
{
AllowTrailingCommas = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNameCaseInsensitive = true,
DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true
};
var updateSubtitleFilesSql = "UPDATE \"SubtitleFiles\" SET \"Title\" = @Title, \"Copy\" = @Copy, \"Language\" = @Language, \"LanguageTags\" = @LanguageTags, \"LastUpdated\" = CURRENT_TIMESTAMP WHERE \"Id\" = @Id";
conn.Execute(updateSubtitleFilesSql, updates, transaction: tran);
}
private static SubtitleTitleInfo CleanSubtitleTitleInfo(string relativePath, string originalFilePath, string path)
{
var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path);
var episodeFileTitle = Path.GetFileNameWithoutExtension(relativePath);
var originalEpisodeFileTitle = Path.GetFileNameWithoutExtension(originalFilePath) ?? string.Empty;
if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase)))
{
Logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle);
subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path);
}
var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !episodeFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList();
if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count)
{
Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle);
subtitleTitleInfo.LanguageTags = cleanedTags;
}
return subtitleTitleInfo;
}
}
}

16
src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs
View File

@ -2,7 +2,6 @@ using System;
using System.IO;
using System.Linq;
using NLog;
using NzbDrone.Common.Instrumentation;
using NzbDrone.Core.Download;
using NzbDrone.Core.Extras.Subtitles;
using NzbDrone.Core.Parser;
@ -12,13 +11,20 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators
{
public class AggregateSubtitleInfo : IAggregateLocalEpisode
{
private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo));
public int Order => 6;
private readonly Logger _logger;
public AggregateSubtitleInfo(Logger logger)
{
_logger = logger;
}
public LocalEpisode Aggregate(LocalEpisode localEpisode, DownloadClientItem downloadClientItem)
{
var path = localEpisode.Path;
var isSubtitleFile = SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(path));
if (!isSubtitleFile)
{
return localEpisode;
@ -31,7 +37,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators
return localEpisode;
}
public static SubtitleTitleInfo CleanSubtitleTitleInfo(EpisodeFile episodeFile, string path)
public SubtitleTitleInfo CleanSubtitleTitleInfo(EpisodeFile episodeFile, string path)
{
var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path);
@ -40,7 +46,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators
if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase)))
{
Logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle);
_logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle);
subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path);
}
@ -49,7 +55,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators
if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count)
{
Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle);
_logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle);
subtitleTitleInfo.LanguageTags = cleanedTags;
}

27
src/NzbDrone.Core/Parser/LanguageParser.cs
View File

@ -33,6 +33,8 @@ namespace NzbDrone.Core.Parser
private static readonly Regex SubtitleLanguageTitleRegex = new Regex(".+?(\\.((?<tags1>full|forced|foreign|default|cc|psdh|sdh)|(?<iso_code>[a-z]{2,3})))*\\.(?<title>[^.]*)(\\.((?<tags2>full|forced|foreign|default|cc|psdh|sdh)|(?<iso_code>[a-z]{2,3})))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex SubtitleTitleRegex = new Regex("((?<title>.+) - )?(?<copy>\\d+)", RegexOptions.Compiled);
public static List<Language> ParseLanguages(string title)
{
foreach (var regex in CleanSeriesTitleRegex)
@ -286,13 +288,36 @@ namespace NzbDrone.Core.Parser
.Select(tag => tag.Value.ToLower());
var rawTitle = matchTitle.Groups["title"].Value;
return new SubtitleTitleInfo
var subtitleTitleInfo = new SubtitleTitleInfo
{
TitleFirst = matchTitle.Groups["tags1"].Captures.Empty(),
LanguageTags = languageTags.ToList(),
RawTitle = rawTitle,
Language = language
};
UpdateTitleAndCopyFromTitle(subtitleTitleInfo);
return subtitleTitleInfo;
}
public static void UpdateTitleAndCopyFromTitle(SubtitleTitleInfo subtitleTitleInfo)
{
if (subtitleTitleInfo.RawTitle is null)
{
subtitleTitleInfo.Title = null;
subtitleTitleInfo.Copy = 0;
}
else if (SubtitleTitleRegex.Match(subtitleTitleInfo.RawTitle) is var match && match.Success)
{
subtitleTitleInfo.Title = match.Groups["title"].Success ? match.Groups["title"].ToString() : null;
subtitleTitleInfo.Copy = int.Parse(match.Groups["copy"].ToString());
}
else
{
subtitleTitleInfo.Title = subtitleTitleInfo.RawTitle;
subtitleTitleInfo.Copy = 0;
}
}
public static List<string> ParseLanguageTags(string fileName)

23
src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs
View File

@ -1,34 +1,15 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using NzbDrone.Core.Languages;
namespace NzbDrone.Core.Parser.Model
{
public class SubtitleTitleInfo
{
private static readonly Regex SubtitleTitleRegex = new Regex("((?<title>.+) - )?(?<copy>\\d+)", RegexOptions.Compiled);
public List<string> LanguageTags { get; set; }
public Language Language { get; set; }
public string RawTitle { get; set; }
private Lazy<(string Title, int Copy)> TitleAndCopy => new Lazy<(string title, int copy)>(() =>
{
if (RawTitle is null)
{
return (null, 0);
}
var match = SubtitleTitleRegex.Match(RawTitle);
if (match.Success)
{
return (match.Groups["title"].Success ? match.Groups["title"].ToString() : null, int.Parse(match.Groups["copy"].ToString()));
}
return (RawTitle, 0);
});
public string Title => TitleAndCopy.Value.Title;
public int Copy => TitleAndCopy.Value.Copy;
public string Title { get; set; }
public int Copy { get; set; }
public bool TitleFirst { get; set; }
}
}