From d898f55660657e9f6cd57ce4659e36c5046cd8f8 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Mon, 8 Feb 2021 00:09:59 +0100 Subject: [PATCH] Generalized RateLimit logic to all indexers based on indexer id --- .../Http/HttpRateLimitKeyFactoryFixture.cs | 30 ---------------- .../TPLTests/RateLimitServiceFixture.cs | 33 +++++++++++++++++ src/NzbDrone.Common/Http/HttpClient.cs | 2 +- .../Http/HttpRateLimitKeyFactory.cs | 28 --------------- src/NzbDrone.Common/Http/HttpRequest.cs | 1 + src/NzbDrone.Common/TPL/RateLimitService.cs | 36 +++++++++++++++++-- .../Download/TorrentClientBase.cs | 1 + .../Download/UsenetClientBase.cs | 4 ++- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 1 + 9 files changed, 73 insertions(+), 63 deletions(-) delete mode 100644 src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs delete mode 100644 src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs diff --git a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs b/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs deleted file mode 100644 index df08a7ef4..000000000 --- a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs +++ /dev/null @@ -1,30 +0,0 @@ -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Common.Http; - -namespace NzbDrone.Common.Test.Http -{ - [TestFixture] - public class HttpRateLimitKeyFactoryFixture - { - [TestCase("http://127.0.0.2:9117/jackett/api/v2.0/indexers/viva/results/torznab/api?t=search&cat=5000,5070,100030,100041", "127.0.0.2:9117/jackett/api/v2.0/indexers/viva")] - public void should_detect_jackett(string url, string expectedKey) - { - var request = new HttpRequest(url); - - var key = HttpRateLimitKeyFactory.GetRateLimitKey(request); - - key.Should().Be(expectedKey); - } - - [TestCase("http://127.0.0.2:9117/jackett", "127.0.0.2")] - public void should_default_to_host(string url, string expectedKey) - { - var request = new HttpRequest(url); - - var key = HttpRateLimitKeyFactory.GetRateLimitKey(request); - - key.Should().Be(expectedKey); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs index e5ccc8244..83f3e95be 100644 --- a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs +++ b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs @@ -88,5 +88,38 @@ namespace NzbDrone.Common.Test.TPLTests (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100)); } + + [Test] + public void should_extend_subkey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400)); + } + + [Test] + public void should_honor_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200)); + } + + [Test] + public void should_not_extend_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200)); + } } } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 10e483076..7e7f72b12 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -111,7 +111,7 @@ namespace NzbDrone.Common.Http if (request.RateLimit != TimeSpan.Zero) { - _rateLimitService.WaitAndPulse(HttpRateLimitKeyFactory.GetRateLimitKey(request), request.RateLimit); + _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit); } _logger.Trace(request); diff --git a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs b/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs deleted file mode 100644 index e4a627e75..000000000 --- a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace NzbDrone.Common.Http -{ - public static class HttpRateLimitKeyFactory - { - // Use a different key for jackett instances to prevent hitting the ratelimit for multiple separate indexers. - private static readonly Regex _regex = new Regex(@"^https?://(.+/jackett/api/v2.0/indexers/\w+)/", RegexOptions.Compiled); - - public static string GetRateLimitKey(HttpRequest request) - { - var match = _regex.Match(request.Url.ToString()); - - if (match.Success) - { - return match.Groups[1].Value; - } - - return request.Url.Host; - } - - } -} diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 2519531ca..355aa6dd6 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -44,6 +44,7 @@ namespace NzbDrone.Common.Http public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } + public string RateLimitKey { get; set; } public Stream ResponseStream { get; set; } public override string ToString() diff --git a/src/NzbDrone.Common/TPL/RateLimitService.cs b/src/NzbDrone.Common/TPL/RateLimitService.cs index 6d1b9ee17..5030ead23 100644 --- a/src/NzbDrone.Common/TPL/RateLimitService.cs +++ b/src/NzbDrone.Common/TPL/RateLimitService.cs @@ -2,12 +2,14 @@ using System.Collections.Concurrent; using NLog; using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; namespace NzbDrone.Common.TPL { public interface IRateLimitService { void WaitAndPulse(string key, TimeSpan interval); + void WaitAndPulse(string key, string subKey, TimeSpan interval); } public class RateLimitService : IRateLimitService @@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL public void WaitAndPulse(string key, TimeSpan interval) { - var waitUntil = _rateLimitStore.AddOrUpdate(key, - (s) => DateTime.UtcNow + interval, - (s,i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval); + WaitAndPulse(key, null, interval); + } + + public void WaitAndPulse(string key, string subKey, TimeSpan interval) + { + var waitUntil = DateTime.UtcNow.Add(interval); + + if (subKey.IsNotNullOrWhiteSpace()) + { + // Expand the base key timer, but don't extend it beyond now+interval. + var baseUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc)); + + if (baseUntil > waitUntil) + { + waitUntil = baseUntil; + } + + // Wait for the full key + var combinedKey = key + "-" + subKey; + waitUntil = _rateLimitStore.AddOrUpdate(combinedKey, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } + else + { + waitUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } waitUntil -= interval; diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index 07f0bd42c..72b47a2db 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download try { var request = new HttpRequest(torrentUrl); + request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString(); request.Headers.Accept = "application/x-bittorrent"; request.AllowAutoRedirect = false; diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index 68135bd22..9dc35ce2f 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -43,7 +43,9 @@ namespace NzbDrone.Core.Download try { - nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData; + var request = new HttpRequest(url); + request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString(); + nzbData = _httpClient.Get(request).ResponseData; _logger.Debug("Downloaded nzb for episode '{0}' finished ({1} bytes from {2})", remoteEpisode.Release.Title, nzbData.Length, url); } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 75310a5e4..d55b3a4a7 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -317,6 +317,7 @@ namespace NzbDrone.Core.Indexers { request.HttpRequest.RateLimit = RateLimit; } + request.HttpRequest.RateLimitKey = Definition.Id.ToString(); return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest)); }