Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generalized RateLimit logic to all indexers based on indexer id

This commit is contained in:
Taloth Saldono 2021-02-08 00:09:59 +01:00
parent a85979c2f6
commit d898f55660
9 changed files with 73 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs
View File

@ -1,30 +0,0 @@
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Common.Http;
namespace NzbDrone.Common.Test.Http
{
[TestFixture]
public class HttpRateLimitKeyFactoryFixture
{
[TestCase("http://127.0.0.2:9117/jackett/api/v2.0/indexers/viva/results/torznab/api?t=search&cat=5000,5070,100030,100041", "127.0.0.2:9117/jackett/api/v2.0/indexers/viva")]
public void should_detect_jackett(string url, string expectedKey)
{
var request = new HttpRequest(url);
var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
key.Should().Be(expectedKey);
}
[TestCase("http://127.0.0.2:9117/jackett", "127.0.0.2")]
public void should_default_to_host(string url, string expectedKey)
{
var request = new HttpRequest(url);
var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
key.Should().Be(expectedKey);
}
}
}

33
src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs
View File

@ -88,5 +88,38 @@ namespace NzbDrone.Common.Test.TPLTests
(GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
}
[Test]
public void should_extend_subkey_delay()
{
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300));
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
(GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400));
}
[Test]
public void should_honor_basekey_delay()
{
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0));
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
(GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200));
}
[Test]
public void should_not_extend_basekey_delay()
{
GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100));
Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
(GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200));
}
}
}

2
src/NzbDrone.Common/Http/HttpClient.cs
View File

@ -111,7 +111,7 @@ namespace NzbDrone.Common.Http
if (request.RateLimit != TimeSpan.Zero)
{
_rateLimitService.WaitAndPulse(HttpRateLimitKeyFactory.GetRateLimitKey(request), request.RateLimit);
_rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit);
}
_logger.Trace(request);

28
src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs
View File

@ -1,28 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace NzbDrone.Common.Http
{
public static class HttpRateLimitKeyFactory
{
// Use a different key for jackett instances to prevent hitting the ratelimit for multiple separate indexers.
private static readonly Regex _regex = new Regex(@"^https?://(.+/jackett/api/v2.0/indexers/\w+)/", RegexOptions.Compiled);
public static string GetRateLimitKey(HttpRequest request)
{
var match = _regex.Match(request.Url.ToString());
if (match.Success)
{
return match.Groups[1].Value;
}
return request.Url.Host;
}
}
}

1
src/NzbDrone.Common/Http/HttpRequest.cs
View File

@ -44,6 +44,7 @@ namespace NzbDrone.Common.Http
public bool StoreResponseCookie { get; set; }
public TimeSpan RequestTimeout { get; set; }
public TimeSpan RateLimit { get; set; }
public string RateLimitKey { get; set; }
public Stream ResponseStream { get; set; }
public override string ToString()

36
src/NzbDrone.Common/TPL/RateLimitService.cs
View File

@ -2,12 +2,14 @@
using System.Collections.Concurrent;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.TPL
{
public interface IRateLimitService
{
void WaitAndPulse(string key, TimeSpan interval);
void WaitAndPulse(string key, string subKey, TimeSpan interval);
}
public class RateLimitService : IRateLimitService
@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL
public void WaitAndPulse(string key, TimeSpan interval)
{
var waitUntil = _rateLimitStore.AddOrUpdate(key,
(s) => DateTime.UtcNow + interval,
(s,i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
WaitAndPulse(key, null, interval);
}
public void WaitAndPulse(string key, string subKey, TimeSpan interval)
{
var waitUntil = DateTime.UtcNow.Add(interval);
if (subKey.IsNotNullOrWhiteSpace())
{
// Expand the base key timer, but don't extend it beyond now+interval.
var baseUntil = _rateLimitStore.AddOrUpdate(key,
(s) => waitUntil,
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc));
if (baseUntil > waitUntil)
{
waitUntil = baseUntil;
}
// Wait for the full key
var combinedKey = key + "-" + subKey;
waitUntil = _rateLimitStore.AddOrUpdate(combinedKey,
(s) => waitUntil,
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
}
else
{
waitUntil = _rateLimitStore.AddOrUpdate(key,
(s) => waitUntil,
(s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
}
waitUntil -= interval;

1
src/NzbDrone.Core/Download/TorrentClientBase.cs
View File

@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download
try
{
var request = new HttpRequest(torrentUrl);
request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString();
request.Headers.Accept = "application/x-bittorrent";
request.AllowAutoRedirect = false;

4
src/NzbDrone.Core/Download/UsenetClientBase.cs
View File

@ -43,7 +43,9 @@ namespace NzbDrone.Core.Download
try
{
nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData;
var request = new HttpRequest(url);
request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString();
nzbData = _httpClient.Get(request).ResponseData;
_logger.Debug("Downloaded nzb for episode '{0}' finished ({1} bytes from {2})", remoteEpisode.Release.Title, nzbData.Length, url);
}

1
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@ -317,6 +317,7 @@ namespace NzbDrone.Core.Indexers
{
request.HttpRequest.RateLimit = RateLimit;
}
request.HttpRequest.RateLimitKey = Definition.Id.ToString();
return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
}