Provider Status housekeeping

Fixed: Clean up indexer status if stored times are in the future
Fixed: Clean up download client status if stored times are in the future
Closes #1396

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimesFixture.cs

@@ -0,0 +1,119 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class FixFutureDownloadClientStatusTimesFixture : CoreTest<FixFutureDownloadClientStatusTimes>
+    {
+        [Test]
+        public void should_set_disabled_till_when_its_too_far_in_the_future()
+        {
+            var disabledTillTime = EscalationBackOff.Periods[1];
+            var downloadClientStatuses = Builder<DownloadClientStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(downloadClientStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<DownloadClientStatus>>(i => i.All(
+                              s => s.DisabledTill.Value < DateTime.UtcNow.AddMinutes(disabledTillTime)))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_set_initial_failure_when_its_in_the_future()
+        {
+            var downloadClientStatuses = Builder<DownloadClientStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(downloadClientStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<DownloadClientStatus>>(i => i.All(
+                              s => s.InitialFailure.Value < DateTime.UtcNow))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_set_most_recent_failure_when_its_in_the_future()
+        {
+            var downloadClientStatuses = Builder<DownloadClientStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(downloadClientStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<DownloadClientStatus>>(i => i.All(
+                              s => s.MostRecentFailure.Value < DateTime.UtcNow))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_not_change_statuses_when_times_are_in_the_past()
+        {
+            var downloadClientStatuses = Builder<DownloadClientStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 0)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(downloadClientStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IDownloadClientStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<DownloadClientStatus>>(i => i.Count == 0)
+                      )
+                  );
+        }
+
+
+    }
+}

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureIndexerStatusTimesFixture.cs

@@ -0,0 +1,119 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Test.Framework;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class FixFutureIndexerStatusTimesFixture : CoreTest<FixFutureIndexerStatusTimes>
+    {
+        [Test]
+        public void should_set_disabled_till_when_its_too_far_in_the_future()
+        {
+            var disabledTillTime = EscalationBackOff.Periods[1];
+            var indexerStatuses = Builder<IndexerStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(indexerStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<IndexerStatus>>(i => i.All(
+                              s => s.DisabledTill.Value < DateTime.UtcNow.AddMinutes(disabledTillTime)))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_set_initial_failure_when_its_in_the_future()
+        {
+            var indexerStatuses = Builder<IndexerStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(indexerStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<IndexerStatus>>(i => i.All(
+                              s => s.InitialFailure.Value < DateTime.UtcNow))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_set_most_recent_failure_when_its_in_the_future()
+        {
+            var indexerStatuses = Builder<IndexerStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5))
+                                                        .With(t => t.EscalationLevel = 1)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(indexerStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<IndexerStatus>>(i => i.All(
+                              s => s.MostRecentFailure.Value < DateTime.UtcNow))
+                      )
+                  );
+        }
+
+        [Test]
+        public void should_not_change_statuses_when_times_are_in_the_past()
+        {
+            var indexerStatuses = Builder<IndexerStatus>.CreateListOfSize(5)
+                                                        .All()
+                                                        .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5))
+                                                        .With(t => t.EscalationLevel = 0)
+                                                        .BuildListOfNew();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Setup(s => s.All())
+                  .Returns(indexerStatuses);
+
+            Subject.Clean();
+
+            Mocker.GetMock<IIndexerStatusRepository>()
+                  .Verify(v => v.UpdateMany(
+                          It.Is<List<IndexerStatus>>(i => i.Count == 0)
+                      )
+                  );
+        }
+
+
+    }
+}

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureRunScheduledTasksFixture.cs

@@ -1,47 +0,0 @@
-using System;
-using FizzWare.NBuilder;
-using FluentAssertions;
-using Microsoft.Practices.ObjectBuilder2;
-using NUnit.Framework;
-using NzbDrone.Core.Housekeeping.Housekeepers;
-using NzbDrone.Core.Jobs;
-using NzbDrone.Core.Test.Framework;
-
-namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
-{
-    [TestFixture]
-    public class FixFutureRunScheduledTasksFixture : DbTest<FixFutureRunScheduledTasks, ScheduledTask>
-    {
-        [Test]
-        public void should_set_last_execution_time_to_now_when_its_in_the_future()
-        {
-            var tasks = Builder<ScheduledTask>.CreateListOfSize(5)
-                                              .All()
-                                              .With(t => t.LastExecution = DateTime.UtcNow.AddDays(5))
-                                              .BuildListOfNew();
-
-            Db.InsertMany(tasks);
-
-            Subject.Clean();
-
-            AllStoredModels.ForEach(t => t.LastExecution.Should().BeBefore(DateTime.UtcNow));
-        }
-
-        [Test]
-        public void should_not_change_last_execution_time_when_its_in_the_past()
-        {
-            var expectedTime = DateTime.UtcNow.AddHours(-1);
-
-            var tasks = Builder<ScheduledTask>.CreateListOfSize(5)
-                                              .All()
-                                              .With(t => t.LastExecution = expectedTime)
-                                              .BuildListOfNew();
-
-            Db.InsertMany(tasks);
-
-            Subject.Clean();
-
-            AllStoredModels.ForEach(t => t.LastExecution.Should().Be(expectedTime));
-        }
-    }
-}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -241,7 +241,8 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupDownloadClientUnavailablePendingReleasesFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupUnusedTagsFixture.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleasesFixture.cs" />
-    <Compile Include="Housekeeping\Housekeepers\FixFutureRunScheduledTasksFixture.cs" />
+    <Compile Include="Housekeeping\Housekeepers\FixFutureDownloadClientStatusTimesFixture.cs" />
+    <Compile Include="Housekeeping\Housekeepers\FixFutureIndexerStatusTimesFixture.cs" />
     <Compile Include="Http\HttpProxySettingsProviderFixture.cs" />
     <Compile Include="Http\TorCacheHttpRequestInterceptorFixture.cs" />
     <Compile Include="IndexerSearchTests\SeriesSearchServiceFixture.cs" />

src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimes.cs

@@ -0,0 +1,12 @@
+using NzbDrone.Core.Download;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class FixFutureDownloadClientStatusTimes : FixFutureProviderStatusTimes<DownloadClientStatus>, IHousekeepingTask
+    {
+        public FixFutureDownloadClientStatusTimes(IDownloadClientStatusRepository downloadClientStatusRepository)
+            : base(downloadClientStatusRepository)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureIndexerStatusTimes.cs

@@ -0,0 +1,12 @@
+using NzbDrone.Core.Indexers;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class FixFutureIndexerStatusTimes : FixFutureProviderStatusTimes<IndexerStatus>, IHousekeepingTask
+    {
+        public FixFutureIndexerStatusTimes(IIndexerStatusRepository indexerStatusRepository)
+            : base(indexerStatusRepository)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureProviderStatusTimes.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public abstract class FixFutureProviderStatusTimes<TModel> where TModel : ProviderStatusBase, new()
+    {
+        private readonly IProviderStatusRepository<TModel> _repo;
+
+        protected FixFutureProviderStatusTimes(IProviderStatusRepository<TModel> repo)
+        {
+            _repo = repo;
+        }
+
+        public void Clean()
+        {
+            var now = DateTime.UtcNow;
+            var statuses = _repo.All().ToList();
+            var toUpdate = new List<TModel>();
+
+            foreach (var status in statuses)
+            {
+                var updated = false;
+                var escalationDelay = EscalationBackOff.Periods[status.EscalationLevel];
+                var disabledTill = now.AddMinutes(escalationDelay);
+
+                if (status.DisabledTill > disabledTill)
+                {
+                    status.DisabledTill = disabledTill;
+                    updated = true;
+                }
+
+                if (status.InitialFailure > now)
+                {
+                    status.InitialFailure = now;
+                    updated = true;
+                }
+
+                if (status.MostRecentFailure > now)
+                {
+                    status.MostRecentFailure = now;
+                    updated = true;
+                }
+
+                if (updated)
+                {
+                    toUpdate.Add(status);
+                }
+            }
+
+            _repo.UpdateMany(toUpdate);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/IndexerStatusService.cs

@@ -1,10 +1,6 @@
-using System;
+using NLog;
-using System.Collections.Generic;
-using System.Linq;
-using NLog;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
-using NzbDrone.Core.ThingiProvider.Events;
 using NzbDrone.Core.ThingiProvider.Status;
 
 namespace NzbDrone.Core.Indexers

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -607,6 +607,9 @@
     <Compile Include="Housekeeping\Housekeepers\CleanupUnusedTags.cs" />
     <Compile Include="Housekeeping\Housekeepers\CleanupOrphanedPendingReleases.cs" />
     <Compile Include="Housekeeping\Housekeepers\DeleteBadMediaCovers.cs" />
+    <Compile Include="Housekeeping\Housekeepers\FixFutureDownloadClientStatusTimes.cs" />
+    <Compile Include="Housekeeping\Housekeepers\FixFutureProviderStatusTimes.cs" />
+    <Compile Include="Housekeeping\Housekeepers\FixFutureIndexerStatusTimes.cs" />
     <Compile Include="Housekeeping\Housekeepers\FixFutureRunScheduledTasks.cs" />
     <Compile Include="Housekeeping\Housekeepers\TrimLogDatabase.cs" />
     <Compile Include="Housekeeping\Housekeepers\UpdateCleanTitleForSeries.cs" />
@@ -1108,6 +1111,7 @@
     <Compile Include="ThingiProvider\ProviderFactory.cs" />
     <Compile Include="ThingiProvider\ProviderMessage.cs" />
     <Compile Include="ThingiProvider\ProviderRepository.cs" />
+    <Compile Include="ThingiProvider\Status\EscalationBackOff.cs" />
     <Compile Include="ThingiProvider\Status\ProviderStatusBase.cs" />
     <Compile Include="ThingiProvider\Status\ProviderStatusRepository.cs" />
     <Compile Include="ThingiProvider\Status\ProviderStatusServiceBase.cs" />

src/NzbDrone.Core/ThingiProvider/Status/EscalationBackOff.cs

@@ -0,0 +1,18 @@
+namespace NzbDrone.Core.ThingiProvider.Status
+{
+    public static class EscalationBackOff
+    {
+        public static readonly int[] Periods =
+        {
+            0,
+            5 * 60,
+            15 * 60,
+            30 * 60,
+            60 * 60,
+            3 * 60 * 60,
+            6 * 60 * 60,
+            12 * 60 * 60,
+            24 * 60 * 60
+        };
+    }
+}

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs

@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.Linq;
 using NLog;
 using NzbDrone.Core.Messaging.Events;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Events;
 
 namespace NzbDrone.Core.ThingiProvider.Status
@@ -21,25 +20,13 @@ namespace NzbDrone.Core.ThingiProvider.Status
         where TProvider : IProvider
         where TModel : ProviderStatusBase, new()
     {
-        private static readonly int[] EscalationBackOffPeriods = {
-                                                                     0,
-                                                                     5 * 60,
-                                                                     15 * 60,
-                                                                     30 * 60,
-                                                                     60 * 60,
-                                                                     3 * 60 * 60,
-                                                                     6 * 60 * 60,
-                                                                     12 * 60 * 60,
-                                                                     24 * 60 * 60
-                                                                 };
-
         protected readonly object _syncRoot = new object();
 
         protected readonly IProviderStatusRepository<TModel> _providerStatusRepository;
         protected readonly IEventAggregator _eventAggregator;
         protected readonly Logger _logger;
 
-        protected int MaximumEscalationLevel { get; set; } = EscalationBackOffPeriods.Length - 1;
+        protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1;
         protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero;
 
         public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
@@ -63,7 +50,7 @@ namespace NzbDrone.Core.ThingiProvider.Status
         {
             var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
 
-            return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
+            return TimeSpan.FromSeconds(EscalationBackOff.Periods[level]);
         }
 
         public virtual void RecordSuccess(int providerId)