Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New: Updated Rarbg request limits 

Closes #5206
This commit is contained in:
Mark McDowall 2023-04-13 19:54:37 -07:00 committed by GitHub
parent ed3d880974
commit 47cf8e6430
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 98 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/NzbDrone.Common/Http/HttpRequestBuilder.cs
View File

@ -20,6 +20,7 @@ namespace NzbDrone.Common.Http
public Dictionary<string, string> Segments { get; private set; }
public HttpHeader Headers { get; private set; }
public bool SuppressHttpError { get; set; }
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
public bool LogHttpError { get; set; }
public bool UseSimplifiedUserAgent { get; set; }
public bool AllowAutoRedirect { get; set; }
@ -102,6 +103,7 @@ namespace NzbDrone.Common.Http
{
request.Method = Method;
request.SuppressHttpError = SuppressHttpError;
request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
request.LogHttpError = LogHttpError;
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
request.AllowAutoRedirect = AllowAutoRedirect;

2
src/NzbDrone.Common/Http/TooManyRequestsException.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
namespace NzbDrone.Common.Http
{

53
src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs
View File

@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Net;
using System.Net.Http;
using FluentAssertions;
using Moq;
@ -86,5 +87,57 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_warn_and_record_failure_on_429_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
}
[Test]
public void should_warn_and_record_failure_on_520_response()
{
Mocker.GetMock<IHttpClient>()
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
var releases = Subject.FetchRecent();
releases.Should().HaveCount(0);
ExceptionVerification.ExpectedWarns(1);
Mocker.GetMock<IIndexerStatusService>()
.Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
}
// Uncomment when RarbgParser is updated
// [Test]
// public void should_warn_and_record_failure_on_200_response_with_rate_limit()
// {
// Mocker.GetMock<IHttpClient>()
// .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
// .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
//
// var releases = Subject.FetchRecent();
//
// releases.Should().HaveCount(0);
//
// ExceptionVerification.ExpectedWarns(1);
//
// Mocker.GetMock<IIndexerStatusService>()
// .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
// }
}
}

11
src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs
View File

@ -1,9 +1,12 @@
using NzbDrone.Common.Exceptions;
using System;
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.Indexers.Exceptions
{
public class RequestLimitReachedException : NzbDroneException
{
public TimeSpan RetryAfter { get; private set; }
public RequestLimitReachedException(string message, params object[] args)
: base(message, args)
{
@ -13,5 +16,11 @@ namespace NzbDrone.Core.Indexers.Exceptions
: base(message)
{
}
public RequestLimitReachedException(string message, TimeSpan retryAfter)
: base(message)
{
RetryAfter = retryAfter;
}
}
}

14
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
@ -243,9 +243,17 @@ namespace NzbDrone.Core.Indexers
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
catch (RequestLimitReachedException ex)
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
if (ex.RetryAfter != TimeSpan.Zero)
{
_indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (ApiKeyException)

4
src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs
View File

@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
@ -18,7 +20,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
public override string Name => "Rarbg";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
: base(httpClient, indexerStatusService, configService, parsingService, logger)

15
src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Net;
using System.Text.RegularExpressions;
using NzbDrone.Common.Http;
@ -17,10 +18,14 @@ namespace NzbDrone.Core.Indexers.Rarbg
switch (indexerResponse.HttpResponse.StatusCode)
{
case HttpStatusCode.TooManyRequests:
throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
case (HttpStatusCode)520:
throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
default:
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
}
break;
@ -44,6 +49,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
if (jsonResponse.Resource.torrent_results == null)
{
// Despite this being the requested behaviour it appears to be problematic, commenting it out for now
// if (jsonResponse.Resource.rate_limit == 1)
// {
// throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
// }
return results;
}

5
src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Net;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
@ -88,6 +89,8 @@ namespace NzbDrone.Core.Indexers.Rarbg
.Resource("/pubapi_v2.php")
.Accept(HttpAccept.Json);
requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
{
requestBuilder.UseSimplifiedUserAgent = true;

1
src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs
View File

@ -7,6 +7,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
{
public string error { get; set; }
public int? error_code { get; set; }
public int? rate_limit { get; set; }
public List<RarbgTorrent> torrent_results { get; set; }
}