From 3e9d0cdb0286c540aae99c2a0f82c458638549d8 Mon Sep 17 00:00:00 2001
From: Bogdan <mynameisbogdan@users.noreply.github.com>
Date: Sat, 9 Dec 2023 17:58:57 +0200
Subject: [PATCH] New: Retry on failed downloads of torrent and nzb files

---
 .../Download/DownloadClientBase.cs            | 40 +++++++++++++++++--
 .../Download/TorrentClientBase.cs             |  4 +-
 .../Download/UsenetClientBase.cs              |  4 +-
 src/NzbDrone.Core/Sonarr.Core.csproj          |  1 +
 4 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/src/NzbDrone.Core/Download/DownloadClientBase.cs b/src/NzbDrone.Core/Download/DownloadClientBase.cs
index 887091891..53318fb68 100644
--- a/src/NzbDrone.Core/Download/DownloadClientBase.cs
+++ b/src/NzbDrone.Core/Download/DownloadClientBase.cs
@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Disk;
+using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Localization;
@@ -11,6 +13,8 @@ using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
+using Polly;
+using Polly.Retry;
 
 namespace NzbDrone.Core.Download
 {
@@ -23,6 +27,37 @@ namespace NzbDrone.Core.Download
         protected readonly Logger _logger;
         protected readonly ILocalizationService _localizationService;
 
+        protected ResiliencePipeline<HttpResponse> RetryStrategy => new ResiliencePipelineBuilder<HttpResponse>()
+            .AddRetry(new RetryStrategyOptions<HttpResponse>
+            {
+                ShouldHandle = static args => args.Outcome switch
+                {
+                    { Result.HasHttpServerError: true } => PredicateResult.True(),
+                    { Result.StatusCode: HttpStatusCode.RequestTimeout } => PredicateResult.True(),
+                    _ => PredicateResult.False()
+                },
+                Delay = TimeSpan.FromSeconds(3),
+                MaxRetryAttempts = 2,
+                BackoffType = DelayBackoffType.Exponential,
+                UseJitter = true,
+                OnRetry = args =>
+                {
+                    var exception = args.Outcome.Exception;
+
+                    if (exception is not null)
+                    {
+                        _logger.Info(exception, "Request for {0} failed with exception '{1}'. Retrying in {2}s.", Definition.Name, exception.Message, args.RetryDelay.TotalSeconds);
+                    }
+                    else
+                    {
+                        _logger.Info("Request for {0} failed with status {1}. Retrying in {2}s.", Definition.Name, args.Outcome.Result?.StatusCode, args.RetryDelay.TotalSeconds);
+                    }
+
+                    return default;
+                }
+            })
+            .Build();
+
         public abstract string Name { get; }
 
         public Type ConfigContract => typeof(TSettings);
@@ -58,10 +93,7 @@ namespace NzbDrone.Core.Download
             return GetType().Name;
         }
 
-        public abstract DownloadProtocol Protocol
-        {
-            get;
-        }
+        public abstract DownloadProtocol Protocol { get; }
 
         public abstract Task<string> Download(RemoteEpisode remoteEpisode, IIndexer indexer);
         public abstract IEnumerable<DownloadClientItem> GetItems();
diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs
index 7f93b3395..dd653968c 100644
--- a/src/NzbDrone.Core/Download/TorrentClientBase.cs
+++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs
@@ -139,7 +139,9 @@ namespace NzbDrone.Core.Download
                 request.Headers.Accept = "application/x-bittorrent";
                 request.AllowAutoRedirect = false;
 
-                var response = await _httpClient.GetAsync(request);
+                var response = await RetryStrategy
+                    .ExecuteAsync(static async (state, _) => await state._httpClient.GetAsync(state.request), (_httpClient, request))
+                    .ConfigureAwait(false);
 
                 if (response.StatusCode == HttpStatusCode.MovedPermanently ||
                     response.StatusCode == HttpStatusCode.Found ||
diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs
index 14d87ef09..5283cc5ca 100644
--- a/src/NzbDrone.Core/Download/UsenetClientBase.cs
+++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs
@@ -49,7 +49,9 @@ namespace NzbDrone.Core.Download
                 var request = indexer?.GetDownloadRequest(url) ?? new HttpRequest(url);
                 request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString();
 
-                var response = await _httpClient.GetAsync(request);
+                var response = await RetryStrategy
+                    .ExecuteAsync(static async (state, _) => await state._httpClient.GetAsync(state.request), (_httpClient, request))
+                    .ConfigureAwait(false);
 
                 nzbData = response.ResponseData;
 
diff --git a/src/NzbDrone.Core/Sonarr.Core.csproj b/src/NzbDrone.Core/Sonarr.Core.csproj
index 94c3a44fe..d7a4c9fa5 100644
--- a/src/NzbDrone.Core/Sonarr.Core.csproj
+++ b/src/NzbDrone.Core/Sonarr.Core.csproj
@@ -7,6 +7,7 @@
     <PackageReference Include="Diacritical.Net" Version="1.0.4" />
     <PackageReference Include="MailKit" Version="3.6.0" />
     <PackageReference Include="Microsoft.AspNetCore.Cryptography.KeyDerivation" Version="6.0.21" />
+    <PackageReference Include="Polly" Version="8.2.0" />
     <PackageReference Include="Servarr.FFMpegCore" Version="4.7.0-26" />
     <PackageReference Include="Servarr.FFprobe" Version="5.1.4.112" />
     <PackageReference Include="System.Memory" Version="4.5.5" />