Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New: Differentiate between short term and long term (more than 6 hours) indexer failures 

Closes #3279
This commit is contained in:
Mark McDowall 2020-10-12 11:24:19 -07:00
parent 7dd64d843a
commit 2adedb97da
5 changed files with 150 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
frontend/src/System/Status/Health/Health.js
View File

@ -18,6 +18,7 @@ function getInternalLink(source) {
case 'IndexerRssCheck':
case 'IndexerSearchCheck':
case 'IndexerStatusCheck':
case 'IndexerLongTermStatusCheck':
return (
<IconButton
name={icons.SETTINGS}

86
src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerLongTermStatusCheckFixture.cs Normal file
View File

@ -0,0 +1,86 @@
using System;
using System.Collections.Generic;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.HealthCheck.Checks;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.HealthCheck.Checks
{
[TestFixture]
public class IndexerLongTermStatusCheckFixture : CoreTest<IndexerLongTermStatusCheck>
{
private List<IIndexer> _indexers = new List<IIndexer>();
private List<IndexerStatus> _blockedIndexers = new List<IndexerStatus>();
[SetUp]
public void SetUp()
{
Mocker.GetMock<IIndexerFactory>()
.Setup(v => v.GetAvailableProviders())
.Returns(_indexers);
Mocker.GetMock<IIndexerStatusService>()
.Setup(v => v.GetBlockedProviders())
.Returns(_blockedIndexers);
}
private Mock<IIndexer> GivenIndexer(int id, double backoffHours, double failureHours)
{
var mockIndexer = new Mock<IIndexer>();
mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id });
mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true);
_indexers.Add(mockIndexer.Object);
if (backoffHours != 0.0)
{
_blockedIndexers.Add(new IndexerStatus
{
ProviderId = id,
InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
EscalationLevel = 5,
DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
});
}
return mockIndexer;
}
[Test]
public void should_not_return_error_when_no_indexers()
{
Subject.Check().ShouldBeOk();
}
[Test]
public void should_return_warning_if_indexer_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
GivenIndexer(2, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
[Test]
public void should_return_error_if_all_indexers_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
Subject.Check().ShouldBeError();
}
[Test]
public void should_return_warning_if_few_indexers_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
GivenIndexer(2, 10.0, 24.0);
GivenIndexer(3, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
}
}
}

8
src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs
View File

@ -59,7 +59,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
[Test]
public void should_return_warning_if_indexer_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
GivenIndexer(1, 2.0, 4.0);
GivenIndexer(2, 0.0, 0.0);
Subject.Check().ShouldBeWarning();
@ -68,7 +68,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
[Test]
public void should_return_error_if_all_indexers_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
GivenIndexer(1, 2.0, 4.0);
Subject.Check().ShouldBeError();
}
@ -76,8 +76,8 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
[Test]
public void should_return_warning_if_few_indexers_unavailable()
{
GivenIndexer(1, 10.0, 24.0);
GivenIndexer(2, 10.0, 24.0);
GivenIndexer(1, 2.0, 4.0);
GivenIndexer(2, 2.0, 4.0);
GivenIndexer(3, 0.0, 0.0);
Subject.Check().ShouldBeWarning();

52
src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using System.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.ThingiProvider.Events;
namespace NzbDrone.Core.HealthCheck.Checks
{
[CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
[CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
[CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
public class IndexerLongTermStatusCheck : HealthCheckBase
{
private readonly IIndexerFactory _providerFactory;
private readonly IIndexerStatusService _providerStatusService;
public IndexerLongTermStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService)
{
_providerFactory = providerFactory;
_providerStatusService = providerStatusService;
}
public override HealthCheck Check()
{
var enabledProviders = _providerFactory.GetAvailableProviders();
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new {Provider = i, Status = s})
.Where(p => p.Status.InitialFailure.HasValue &&
p.Status.InitialFailure.Value.Before(
DateTime.UtcNow.AddHours(-6)))
.ToList();
if (backOffProviders.Empty())
{
return new HealthCheck(GetType());
}
if (backOffProviders.Count == enabledProviders.Count)
{
return new HealthCheck(GetType(), HealthCheckResult.Error,
"All indexers are unavailable due to failures for more than 6 hours", "#indexers-are-unavailable-due-to-failures");
}
return new HealthCheck(GetType(), HealthCheckResult.Warning,
string.Format("Indexers unavailable due to failures for more than 6 hours: {0}",
string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
"#indexers-are-unavailable-due-to-failures");
}
}
}

11
src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs
View File

@ -24,10 +24,13 @@ namespace NzbDrone.Core.HealthCheck.Checks
{
var enabledProviders = _providerFactory.GetAvailableProviders();
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new { Provider = i, Status = s })
.ToList();
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new {Provider = i, Status = s})
.Where(p => p.Status.InitialFailure.HasValue &&
p.Status.InitialFailure.Value.After(
DateTime.UtcNow.AddHours(-6)))
.ToList();
if (backOffProviders.Empty())
{