From 20af2c8c0f18c1f1d3f8d915e04e3401ea3684f0 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 12 Aug 2017 09:40:15 +0200 Subject: [PATCH] Fixed: Releases no longer available on the indexer should be removed from the pending queue. fixes #679 --- .../DownloadApprovedFixture.cs | 24 +++++++++++++++- .../Download/DownloadServiceFixture.cs | 15 ++++++++++ .../RemoveGrabbedFixture.cs | 2 +- src/NzbDrone.Core/Download/DownloadService.cs | 5 ++++ .../Download/ProcessDownloadDecisions.cs | 9 +++++- .../Download/TorrentClientBase.cs | 10 +++++-- .../Download/UsenetClientBase.cs | 8 +++++- .../Exceptions/ReleaseUnavailableException.cs | 28 +++++++++++++++++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 9 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs diff --git a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs index 6100e7590..32035897a 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles; @@ -178,7 +179,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests } [Test] - public void should_return_an_empty_list_when_none_are_appproved() + public void should_return_an_empty_list_when_none_are_approved() { var decisions = new List(); decisions.Add(new DownloadDecision(null, new Rejection("Failure!"))); @@ -263,5 +264,26 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet)), Times.Once()); Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Torrent)), Times.Once()); } + + [Test] + public void should_add_to_rejected_if_release_unavailable_on_indexer() + { + var episodes = new List { GetEpisode(1) }; + var remoteEpisode = GetRemoteEpisode(episodes, new QualityModel(Quality.HDTV720p)); + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteEpisode)); + + Mocker.GetMock() + .Setup(s => s.DownloadReport(It.IsAny())) + .Throws(new ReleaseUnavailableException(remoteEpisode.Release, "That 404 Error is not just a Quirk")); + + var result = Subject.ProcessDecisions(decisions); + + result.Grabbed.Should().BeEmpty(); + result.Rejected.Should().NotBeEmpty(); + + ExceptionVerification.ExpectedWarns(1); + } } } diff --git a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs index 330235ea1..6d6f681c6 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadServiceFixture.cs @@ -180,6 +180,21 @@ namespace NzbDrone.Core.Test.Download .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Never()); } + [Test] + public void Download_report_should_not_trigger_indexer_backoff_on_indexer_404_error() + { + var mock = WithUsenetClient(); + mock.Setup(s => s.Download(It.IsAny())) + .Callback(v => { + throw new ReleaseUnavailableException(v.Release, "Error", new WebException()); + }); + + Assert.Throws(() => Subject.DownloadReport(_parseResult)); + + Mocker.GetMock() + .Verify(v => v.RecordFailure(It.IsAny(), It.IsAny()), Times.Never()); + } + [Test] public void should_not_attempt_download_if_client_isnt_configured() { diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs index b70f24fdc..b988a30c9 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests _remoteEpisode.Series = _series; _remoteEpisode.ParsedEpisodeInfo = _parsedEpisodeInfo; _remoteEpisode.Release = _release; - + _temporarilyRejected = new DownloadDecision(_remoteEpisode, new Rejection("Temp Rejected", RejectionType.Temporary)); Mocker.GetMock() diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index f0837b4c5..db90dce16 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -75,6 +75,11 @@ namespace NzbDrone.Core.Download _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id); _indexerStatusService.RecordSuccess(remoteEpisode.Release.IndexerId); } + catch (ReleaseUnavailableException) + { + _logger.Trace("Release {0} no longer available on indexer.", remoteEpisode); + throw; + } catch (ReleaseDownloadException ex) { var http429 = ex.InnerException as TooManyRequestsException; diff --git a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs index bd146dff7..6fb248bf0 100644 --- a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs +++ b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs @@ -6,6 +6,7 @@ using NLog; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; namespace NzbDrone.Core.Download @@ -40,6 +41,7 @@ namespace NzbDrone.Core.Download var grabbed = new List(); var pending = new List(); var failed = new List(); + var rejected = decisions.Where(d => d.Rejected).ToList(); var usenetFailed = false; var torrentFailed = false; @@ -74,6 +76,11 @@ namespace NzbDrone.Core.Download _downloadService.DownloadReport(remoteEpisode); grabbed.Add(report); } + catch (ReleaseUnavailableException) + { + _logger.Warn("Failed to download release from indexer, no longer available. " + remoteEpisode); + rejected.Add(report); + } catch (Exception ex) { if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException) @@ -99,7 +106,7 @@ namespace NzbDrone.Core.Download pending.AddRange(ProcessFailedGrabs(grabbed, failed)); - return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList()); + return new ProcessedDecisions(grabbed, pending, rejected); } internal List GetQualifiedReports(IEnumerable decisions) diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index bcfd11f7a..07f0bd42c 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -33,7 +33,7 @@ namespace NzbDrone.Core.Download _httpClient = httpClient; _torrentFileInfoReader = torrentFileInfoReader; } - + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public virtual bool PreferTorrentFile => false; @@ -61,7 +61,7 @@ namespace NzbDrone.Core.Download { magnetUrl = torrentInfo.MagnetUrl; } - + if (PreferTorrentFile) { if (torrentUrl.IsNotNullOrWhiteSpace()) @@ -160,6 +160,12 @@ namespace NzbDrone.Core.Download } catch (HttpException ex) { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading torrent file for episode '{0}' failed since it no longer exists ({1})", remoteEpisode.Release.Title, torrentUrl); + throw new ReleaseUnavailableException(remoteEpisode.Release, "Downloading torrent failed", ex); + } + if ((int)ex.Response.StatusCode == 429) { _logger.Error("API Grab Limit reached for {0}", torrentUrl); diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index a6c0ed7d5..f42d40ac6 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -26,7 +26,7 @@ namespace NzbDrone.Core.Download { _httpClient = httpClient; } - + public override DownloadProtocol Protocol => DownloadProtocol.Usenet; protected abstract string AddFromNzbFile(RemoteEpisode remoteEpisode, string filename, byte[] fileContent); @@ -46,6 +46,12 @@ namespace NzbDrone.Core.Download } catch (HttpException ex) { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading nzb file for episode '{0}' failed since it no longer exists ({1})", remoteEpisode.Release.Title, url); + throw new ReleaseUnavailableException(remoteEpisode.Release, "Downloading torrent failed", ex); + } + if ((int)ex.Response.StatusCode == 429) { _logger.Error("API Grab Limit reached for {0}", url); diff --git a/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs new file mode 100644 index 000000000..41442c1ea --- /dev/null +++ b/src/NzbDrone.Core/Exceptions/ReleaseUnavailableException.cs @@ -0,0 +1,28 @@ +using System; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Exceptions +{ + public class ReleaseUnavailableException : ReleaseDownloadException + { + public ReleaseUnavailableException(ReleaseInfo release, string message, params object[] args) + : base(release, message, args) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message) + : base(release, message) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException, params object[] args) + : base(release, message, innerException, args) + { + } + + public ReleaseUnavailableException(ReleaseInfo release, string message, Exception innerException) + : base(release, message, innerException) + { + } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 364d8b821..6c5ae0089 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -539,6 +539,7 @@ +