From 381834edcefcf61464b1833f1769ed571f07703e Mon Sep 17 00:00:00 2001 From: Qstick Date: Thu, 24 Nov 2022 21:15:20 -0600 Subject: [PATCH] New: IMDb List Support --- .../ImportListSyncServiceFixture.cs | 37 +++++++++---- .../SkyHook/SkyHookProxySearchFixture.cs | 27 +++++++++- .../FetchAndParseImportListService.cs | 4 +- .../ImportLists/HttpImportListBase.cs | 4 +- .../ImportLists/Imdb/ImdbListImport.cs | 49 +++++++++++++++++ .../ImportLists/Imdb/ImdbListParser.cs | 53 +++++++++++++++++++ .../Imdb/ImdbListRequestGenerator.cs | 23 ++++++++ .../ImportLists/Imdb/ImdbListSettings.cs | 35 ++++++++++++ .../ImportLists/ImportListSyncService.cs | 34 ++++++++---- .../MetadataSource/ISearchForNewSeries.cs | 3 +- .../MetadataSource/SkyHook/SkyHookProxy.cs | 14 +++++ src/NzbDrone.Core/Parser/Parser.cs | 18 +++++++ src/NzbDrone.Core/Tv/SeriesRepository.cs | 9 ++++ src/NzbDrone.Core/Tv/SeriesService.cs | 6 +++ 14 files changed, 290 insertions(+), 26 deletions(-) create mode 100644 src/NzbDrone.Core/ImportLists/Imdb/ImdbListImport.cs create mode 100644 src/NzbDrone.Core/ImportLists/Imdb/ImdbListParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/Imdb/ImdbListRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/Imdb/ImdbListSettings.cs diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs index 2822d9838..da938b34e 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -25,17 +25,21 @@ namespace NzbDrone.Core.Test.ImportListTests _importListReports = new List { importListItem1 }; - Mocker.GetMock() - .Setup(v => v.Fetch()) - .Returns(_importListReports); + Mocker.GetMock() + .Setup(v => v.AllSeriesTvdbIds()) + .Returns(new List()); Mocker.GetMock() .Setup(v => v.SearchForNewSeries(It.IsAny())) .Returns(new List()); + Mocker.GetMock() + .Setup(v => v.SearchForNewSeriesByImdbId(It.IsAny())) + .Returns(new List()); + Mocker.GetMock() - .Setup(v => v.Get(It.IsAny())) - .Returns(new ImportListDefinition { ShouldMonitor = MonitorTypes.All }); + .Setup(v => v.All()) + .Returns(new List { new ImportListDefinition { ShouldMonitor = MonitorTypes.All } }); Mocker.GetMock() .Setup(v => v.Fetch()) @@ -51,11 +55,16 @@ namespace NzbDrone.Core.Test.ImportListTests _importListReports.First().TvdbId = 81189; } + private void WithImdbId() + { + _importListReports.First().ImdbId = "tt0496424"; + } + private void WithExistingSeries() { Mocker.GetMock() - .Setup(v => v.FindByTvdbId(_importListReports.First().TvdbId)) - .Returns(new Series { TvdbId = _importListReports.First().TvdbId }); + .Setup(v => v.AllSeriesTvdbIds()) + .Returns(new List { _importListReports.First().TvdbId }); } private void WithExcludedSeries() @@ -74,8 +83,8 @@ namespace NzbDrone.Core.Test.ImportListTests private void WithMonitorType(MonitorTypes monitor) { Mocker.GetMock() - .Setup(v => v.Get(It.IsAny())) - .Returns(new ImportListDefinition { ShouldMonitor = monitor }); + .Setup(v => v.All()) + .Returns(new List { new ImportListDefinition { ShouldMonitor = monitor } }); } [Test] @@ -97,6 +106,16 @@ namespace NzbDrone.Core.Test.ImportListTests .Verify(v => v.SearchForNewSeries(It.IsAny()), Times.Never()); } + [Test] + public void should_search_by_imdb_if_series_title_and_series_imdb() + { + WithImdbId(); + Subject.Execute(new ImportListSyncCommand()); + + Mocker.GetMock() + .Verify(v => v.SearchForNewSeriesByImdbId(It.IsAny()), Times.Once()); + } + [Test] public void should_not_add_if_existing_series() { diff --git a/src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxySearchFixture.cs b/src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxySearchFixture.cs index 5248b9ecc..4200f4547 100644 --- a/src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxySearchFixture.cs +++ b/src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxySearchFixture.cs @@ -1,6 +1,7 @@ -using FluentAssertions; +using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Core.MetadataSource; using NzbDrone.Core.MetadataSource.SkyHook; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.Tv; @@ -42,6 +43,30 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook ExceptionVerification.IgnoreWarns(); } + [TestCase("tt0496424", "30 Rock")] + public void should_search_by_imdb(string title, string expected) + { + var result = Subject.SearchForNewSeriesByImdbId(title); + + result.Should().NotBeEmpty(); + + result[0].Title.Should().Be(expected); + + ExceptionVerification.IgnoreWarns(); + } + + [TestCase("4565se")] + public void should_not_search_by_imdb_if_invalid(string title) + { + var result = Subject.SearchForNewSeriesByImdbId(title); + result.Should().BeEmpty(); + + Mocker.GetMock() + .Verify(v => v.SearchForNewSeries(It.IsAny()), Times.Never()); + + ExceptionVerification.IgnoreWarns(); + } + [TestCase("tvdbid:")] [TestCase("tvdbid: 99999999999999999999")] [TestCase("tvdbid: 0")] diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index 951e628eb..7409b2851 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -71,7 +71,7 @@ namespace NzbDrone.Core.ImportLists Task.WaitAll(taskList.ToArray()); - result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList(); + result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList(); _logger.Debug("Found {0} reports", result.Count); @@ -118,7 +118,7 @@ namespace NzbDrone.Core.ImportLists Task.WaitAll(taskList.ToArray()); - result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList(); + result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList(); return result; } diff --git a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs index 7100b6882..f350853de 100644 --- a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs @@ -165,9 +165,9 @@ namespace NzbDrone.Core.ImportLists return CleanupListItems(releases); } - protected virtual bool IsValidItem(ImportListItemInfo release) + protected virtual bool IsValidItem(ImportListItemInfo listItem) { - if (release.Title.IsNullOrWhiteSpace()) + if (listItem.Title.IsNullOrWhiteSpace() && listItem.ImdbId.IsNullOrWhiteSpace() && listItem.TmdbId == 0) { return false; } diff --git a/src/NzbDrone.Core/ImportLists/Imdb/ImdbListImport.cs b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListImport.cs new file mode 100644 index 000000000..ca593f3fc --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListImport.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.ImportLists.Imdb +{ + public class ImdbListImport : HttpImportListBase + { + public override string Name => "IMDb Lists"; + + public override ImportListType ListType => ImportListType.Other; + + public ImdbListImport(IHttpClient httpClient, + IImportListStatusService importListStatusService, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + } + + public override IEnumerable DefaultDefinitions + { + get + { + foreach (var def in base.DefaultDefinitions) + { + yield return def; + } + } + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new ImdbListRequestGenerator() + { + Settings = Settings + }; + } + + public override IParseImportListResponse GetParser() + { + return new ImdbListParser(); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Imdb/ImdbListParser.cs b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListParser.cs new file mode 100644 index 000000000..75e5e6bf6 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListParser.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.Imdb +{ + public class ImdbListParser : IParseImportListResponse + { + public IList ParseResponse(ImportListResponse importListResponse) + { + var importResponse = importListResponse; + + var series = new List(); + + if (!PreProcess(importResponse)) + { + return series; + } + + // Parse TSV response from IMDB export + var rows = importResponse.Content.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + + series = rows.Skip(1).SelectList(m => m.Split(',')).Where(m => m.Length > 1).SelectList(i => new ImportListItemInfo { ImdbId = i[1] }); + + return series; + } + + protected virtual bool PreProcess(ImportListResponse listResponse) + { + if (listResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(listResponse, + "Imdb call resulted in an unexpected StatusCode [{0}]", + listResponse.HttpResponse.StatusCode); + } + + if (listResponse.HttpResponse.Headers.ContentType != null && + listResponse.HttpResponse.Headers.ContentType.Contains("text/json") && + listResponse.HttpRequest.Headers.Accept != null && + !listResponse.HttpRequest.Headers.Accept.Contains("text/json")) + { + throw new ImportListException(listResponse, + "Imdb responded with html content. Site is likely blocked or unavailable."); + } + + return true; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Imdb/ImdbListRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListRequestGenerator.cs new file mode 100644 index 000000000..78c31eb29 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListRequestGenerator.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.Imdb +{ + public class ImdbListRequestGenerator : IImportListRequestGenerator + { + public ImdbListSettings Settings { get; set; } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + var httpRequest = new HttpRequest($"https://www.imdb.com/list/{Settings.ListId}/export", new HttpAccept("*/*")); + var request = new ImportListRequest(httpRequest.Url.ToString(), new HttpAccept(httpRequest.Headers.Accept)); + + request.HttpRequest.SuppressHttpError = true; + + pageableRequests.Add(new List { request }); + + return pageableRequests; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Imdb/ImdbListSettings.cs b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListSettings.cs new file mode 100644 index 000000000..72ca21cc7 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Imdb/ImdbListSettings.cs @@ -0,0 +1,35 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.Imdb +{ + public class ImdbSettingsValidator : AbstractValidator + { + public ImdbSettingsValidator() + { + RuleFor(c => c.ListId) + .Matches(@"^ls\d+$") + .WithMessage("List ID mist be an IMDb List ID of the form 'ls12345678'"); + } + } + + public class ImdbListSettings : IImportListSettings + { + private static readonly ImdbSettingsValidator Validator = new ImdbSettingsValidator(); + + public ImdbListSettings() + { + } + + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "List ID", HelpText = "IMDb list ID (e.g ls12345678)")] + public string ListId { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index 4c00afe52..c6201e4ee 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -69,6 +69,8 @@ namespace NzbDrone.Core.ImportLists var reportNumber = 1; var listExclusions = _importListExclusionService.All(); + var importLists = _importListFactory.All(); + var existingTvdbIds = _seriesService.AllSeriesTvdbIds(); foreach (var report in reports) { @@ -76,7 +78,20 @@ namespace NzbDrone.Core.ImportLists reportNumber++; - var importList = _importListFactory.Get(report.ImportListId); + var importList = importLists.Single(x => x.Id == report.ImportListId); + + // Map by IMDbId if we have it + if (report.TvdbId <= 0 && report.ImdbId.IsNotNullOrWhiteSpace()) + { + var mappedSeries = _seriesSearchService.SearchForNewSeriesByImdbId(report.ImdbId) + .FirstOrDefault(); + + if (mappedSeries != null) + { + report.TvdbId = mappedSeries.TvdbId; + report.Title = mappedSeries?.Title; + } + } // Map TVDb if we only have a series name if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace()) @@ -91,16 +106,6 @@ namespace NzbDrone.Core.ImportLists } } - // Check to see if series in DB - var existingSeries = _seriesService.FindByTvdbId(report.TvdbId); - - // Break if Series Exists in DB - if (existingSeries != null) - { - _logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title); - continue; - } - // Check to see if series excluded var excludedSeries = listExclusions.Where(s => s.TvdbId == report.TvdbId).SingleOrDefault(); @@ -110,6 +115,13 @@ namespace NzbDrone.Core.ImportLists continue; } + // Break if Series Exists in DB + if (existingTvdbIds.Any(x => x == report.TvdbId)) + { + _logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title); + continue; + } + // Append Series if not already in DB or already on add list if (seriesToAdd.All(s => s.TvdbId != report.TvdbId)) { diff --git a/src/NzbDrone.Core/MetadataSource/ISearchForNewSeries.cs b/src/NzbDrone.Core/MetadataSource/ISearchForNewSeries.cs index bd6698251..d67d6bd15 100644 --- a/src/NzbDrone.Core/MetadataSource/ISearchForNewSeries.cs +++ b/src/NzbDrone.Core/MetadataSource/ISearchForNewSeries.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using NzbDrone.Core.Tv; namespace NzbDrone.Core.MetadataSource @@ -6,5 +6,6 @@ namespace NzbDrone.Core.MetadataSource public interface ISearchForNewSeries { List SearchForNewSeries(string title); + List SearchForNewSeriesByImdbId(string imdbId); } } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index 3552a0aa3..d5ba41849 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -69,6 +69,20 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return new Tuple>(series, episodes.ToList()); } + public List SearchForNewSeriesByImdbId(string imdbId) + { + imdbId = Parser.Parser.NormalizeImdbId(imdbId); + + if (imdbId == null) + { + return new List(); + } + + var results = SearchForNewSeries($"imdb:{imdbId}"); + + return results; + } + public List SearchForNewSeries(string title) { try diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index cd352c5d0..111687d22 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -733,6 +733,24 @@ namespace NzbDrone.Core.Parser return title.Trim().ToLower(); } + public static string NormalizeImdbId(string imdbId) + { + var imdbRegex = new Regex(@"^(\d{1,10}|(tt)\d{1,10})$"); + + if (!imdbRegex.IsMatch(imdbId)) + { + return null; + } + + if (imdbId.Length > 2) + { + imdbId = imdbId.Replace("tt", "").PadLeft(7, '0'); + return $"tt{imdbId}"; + } + + return null; + } + public static string ParseReleaseGroup(string title) { title = title.Trim(); diff --git a/src/NzbDrone.Core/Tv/SeriesRepository.cs b/src/NzbDrone.Core/Tv/SeriesRepository.cs index 586d04c86..045f76281 100644 --- a/src/NzbDrone.Core/Tv/SeriesRepository.cs +++ b/src/NzbDrone.Core/Tv/SeriesRepository.cs @@ -16,6 +16,7 @@ namespace NzbDrone.Core.Tv Series FindByTvdbId(int tvdbId); Series FindByTvRageId(int tvRageId); Series FindByPath(string path); + List AllSeriesTvdbIds(); Dictionary AllSeriesPaths(); } @@ -73,6 +74,14 @@ namespace NzbDrone.Core.Tv .FirstOrDefault(); } + public List AllSeriesTvdbIds() + { + using (var conn = _database.OpenConnection()) + { + return conn.Query("SELECT TvdbId FROM Series").ToList(); + } + } + public Dictionary AllSeriesPaths() { using (var conn = _database.OpenConnection()) diff --git a/src/NzbDrone.Core/Tv/SeriesService.cs b/src/NzbDrone.Core/Tv/SeriesService.cs index a4825be85..2bb4976ee 100644 --- a/src/NzbDrone.Core/Tv/SeriesService.cs +++ b/src/NzbDrone.Core/Tv/SeriesService.cs @@ -24,6 +24,7 @@ namespace NzbDrone.Core.Tv Series FindByPath(string path); void DeleteSeries(int seriesId, bool deleteFiles, bool addImportListExclusion); List GetAllSeries(); + List AllSeriesTvdbIds(); Dictionary GetAllSeriesPaths(); List AllForTag(int tagId); Series UpdateSeries(Series series, bool updateEpisodesToMatchSeason = true, bool publishUpdatedEvent = true); @@ -160,6 +161,11 @@ namespace NzbDrone.Core.Tv return _seriesRepository.All().ToList(); } + public List AllSeriesTvdbIds() + { + return _seriesRepository.AllSeriesTvdbIds().ToList(); + } + public Dictionary GetAllSeriesPaths() { return _seriesRepository.AllSeriesPaths();