New: Differentiate between short term and long term (more than 6 hours) indexer failures

Closes #3279

frontend/src/System/Status/Health/Health.js

@@ -18,6 +18,7 @@ function getInternalLink(source) {
     case 'IndexerRssCheck':
     case 'IndexerSearchCheck':
     case 'IndexerStatusCheck':
+    case 'IndexerLongTermStatusCheck':
       return (
         <IconButton
           name={icons.SETTINGS}

src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerLongTermStatusCheckFixture.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.HealthCheck.Checks;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.HealthCheck.Checks
+{
+    [TestFixture]
+    public class IndexerLongTermStatusCheckFixture : CoreTest<IndexerLongTermStatusCheck>
+    {
+        private List<IIndexer> _indexers = new List<IIndexer>();
+        private List<IndexerStatus> _blockedIndexers = new List<IndexerStatus>();
+
+        [SetUp]
+        public void SetUp()
+        {
+            Mocker.GetMock<IIndexerFactory>()
+                  .Setup(v => v.GetAvailableProviders())
+                  .Returns(_indexers);
+
+            Mocker.GetMock<IIndexerStatusService>()
+                   .Setup(v => v.GetBlockedProviders())
+                   .Returns(_blockedIndexers);
+        }
+
+        private Mock<IIndexer> GivenIndexer(int id, double backoffHours, double failureHours)
+        {
+            var mockIndexer = new Mock<IIndexer>();
+            mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id });
+            mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true);
+
+            _indexers.Add(mockIndexer.Object);
+
+            if (backoffHours != 0.0)
+            {
+                _blockedIndexers.Add(new IndexerStatus
+                    {
+                        ProviderId = id,
+                        InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
+                        MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
+                        EscalationLevel = 5,
+                        DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
+                    });
+            }
+
+            return mockIndexer;
+        }
+
+
+        [Test]
+        public void should_not_return_error_when_no_indexers()
+        {
+            Subject.Check().ShouldBeOk();
+        }
+
+        [Test]
+        public void should_return_warning_if_indexer_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(2, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+
+        [Test]
+        public void should_return_error_if_all_indexers_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+
+            Subject.Check().ShouldBeError();
+        }
+
+        [Test]
+        public void should_return_warning_if_few_indexers_unavailable()
+        {
+            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(2, 10.0, 24.0);
+            GivenIndexer(3, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+    }
+}

src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs

@@ -59,7 +59,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
         [Test]
         public void should_return_warning_if_indexer_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
             GivenIndexer(2, 0.0, 0.0);
 
             Subject.Check().ShouldBeWarning();
@@ -68,7 +68,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
         [Test]
         public void should_return_error_if_all_indexers_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
 
             Subject.Check().ShouldBeError();
         }
@@ -76,8 +76,8 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
         [Test]
         public void should_return_warning_if_few_indexers_unavailable()
         {
-            GivenIndexer(1, 10.0, 24.0);
+            GivenIndexer(1, 2.0, 4.0);
-            GivenIndexer(2, 10.0, 24.0);
+            GivenIndexer(2, 2.0, 4.0);
             GivenIndexer(3, 0.0, 0.0);
 
             Subject.Check().ShouldBeWarning();

src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderUpdatedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<IIndexer>))]
+    public class IndexerLongTermStatusCheck : HealthCheckBase
+    {
+        private readonly IIndexerFactory _providerFactory;
+        private readonly IIndexerStatusService _providerStatusService;
+
+        public IndexerLongTermStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService)
+        {
+            _providerFactory = providerFactory;
+            _providerStatusService = providerStatusService;
+        }
+
+        public override HealthCheck Check()
+        {
+            var enabledProviders = _providerFactory.GetAvailableProviders();
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
+                                                       i => i.Definition.Id,
+                                                       s => s.ProviderId,
+                                                       (i, s) => new {Provider = i, Status = s})
+                                                   .Where(p => p.Status.InitialFailure.HasValue &&
+                                                               p.Status.InitialFailure.Value.Before(
+                                                                   DateTime.UtcNow.AddHours(-6)))
+                                                   .ToList();
+
+            if (backOffProviders.Empty())
+            {
+                return new HealthCheck(GetType());
+            }
+
+            if (backOffProviders.Count == enabledProviders.Count)
+            {
+                return new HealthCheck(GetType(), HealthCheckResult.Error,
+                    "All indexers are unavailable due to failures for more than 6 hours", "#indexers-are-unavailable-due-to-failures");
+            }
+
+            return new HealthCheck(GetType(), HealthCheckResult.Warning,
+                string.Format("Indexers unavailable due to failures for more than 6 hours: {0}",
+                    string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
+                "#indexers-are-unavailable-due-to-failures");
+        }
+    }
+}

src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs

@@ -24,10 +24,13 @@ namespace NzbDrone.Core.HealthCheck.Checks
         {
             var enabledProviders = _providerFactory.GetAvailableProviders();
             var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
-                    i => i.Definition.Id,
+                                                       i => i.Definition.Id,
-                    s => s.ProviderId,
+                                                       s => s.ProviderId,
-                    (i, s) => new { Provider = i, Status = s })
+                                                       (i, s) => new {Provider = i, Status = s})
-                .ToList();
+                                                   .Where(p => p.Status.InitialFailure.HasValue &&
+                                                               p.Status.InitialFailure.Value.After(
+                                                                   DateTime.UtcNow.AddHours(-6)))
+                                                   .ToList();
 
             if (backOffProviders.Empty())
             {