Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored IndexerStatusService into Thingy Provider architecture.

This commit is contained in:
Taloth Saldono 2017-05-12 21:41:05 +02:00
parent 9f8091e4d7
commit f4bea5512c
15 changed files with 248 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs
View File

@ -27,7 +27,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
public void should_not_ignore_pending_items_from_available_indexer()
{
Mocker.GetMock<IIndexerStatusService>()
.Setup(v => v.GetBlockedIndexers())
.Setup(v => v.GetBlockedProviders())
.Returns(new List<IndexerStatus>());
GivenPendingRelease();
@ -43,7 +43,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests
public void should_ignore_pending_items_from_unavailable_indexer()
{
Mocker.GetMock<IIndexerStatusService>()
.Setup(v => v.GetBlockedIndexers())
.Setup(v => v.GetBlockedProviders())
.Returns(new List<IndexerStatus> { new IndexerStatus { ProviderId = 1, DisabledTill = DateTime.UtcNow.AddHours(2) } });
GivenPendingRelease();

2
src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs
View File

@ -22,7 +22,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks
.Returns(_indexers);
Mocker.GetMock<IIndexerStatusService>()
.Setup(v => v.GetBlockedIndexers())
.Setup(v => v.GetBlockedProviders())
.Returns(_blockedIndexers);
}

10
src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs
View File

@ -11,7 +11,7 @@ namespace NzbDrone.Core.Test.IndexerTests
public class IndexerStatusServiceFixture : CoreTest<IndexerStatusService>
{
private DateTime _epoch;
[SetUp]
public void SetUp()
{
@ -21,7 +21,7 @@ namespace NzbDrone.Core.Test.IndexerTests
private void WithStatus(IndexerStatus status)
{
Mocker.GetMock<IIndexerStatusRepository>()
.Setup(v => v.FindByIndexerId(1))
.Setup(v => v.FindByProviderId(1))
.Returns(status);
Mocker.GetMock<IIndexerStatusRepository>()
@ -44,7 +44,7 @@ namespace NzbDrone.Core.Test.IndexerTests
VerifyUpdate();
var status = Subject.GetBlockedIndexers().FirstOrDefault();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().NotBeNull();
status.DisabledTill.Should().HaveValue();
status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
@ -59,7 +59,7 @@ namespace NzbDrone.Core.Test.IndexerTests
VerifyUpdate();
var status = Subject.GetBlockedIndexers().FirstOrDefault();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().BeNull();
}
@ -82,7 +82,7 @@ namespace NzbDrone.Core.Test.IndexerTests
Subject.RecordSuccess(1);
Subject.RecordFailure(1);
var status = Subject.GetBlockedIndexers().FirstOrDefault();
var status = Subject.GetBlockedProviders().FirstOrDefault();
status.Should().NotBeNull();
status.DisabledTill.Should().HaveValue();
status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);

2
src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs
View File

@ -101,7 +101,7 @@ namespace NzbDrone.Core.Download.Pending
private List<ReleaseInfo> FilterBlockedIndexers(List<ReleaseInfo> releases)
{
var blockedIndexers = new HashSet<int>(_indexerStatusService.GetBlockedIndexers().Select(v => v.ProviderId));
var blockedIndexers = new HashSet<int>(_indexerStatusService.GetBlockedProviders().Select(v => v.ProviderId));
return releases.Where(release => !blockedIndexers.Contains(release.IndexerId)).ToList();
}

22
src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs
View File

@ -7,36 +7,36 @@ namespace NzbDrone.Core.HealthCheck.Checks
{
public class IndexerStatusCheck : HealthCheckBase
{
private readonly IIndexerFactory _indexerFactory;
private readonly IIndexerStatusService _indexerStatusService;
private readonly IIndexerFactory _providerFactory;
private readonly IIndexerStatusService _providerStatusService;
public IndexerStatusCheck(IIndexerFactory indexerFactory, IIndexerStatusService indexerStatusService)
public IndexerStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService)
{
_indexerFactory = indexerFactory;
_indexerStatusService = indexerStatusService;
_providerFactory = providerFactory;
_providerStatusService = providerStatusService;
}
public override HealthCheck Check()
{
var enabledIndexers = _indexerFactory.GetAvailableProviders();
var backOffIndexers = enabledIndexers.Join(_indexerStatusService.GetBlockedIndexers(),
var enabledProviders = _providerFactory.GetAvailableProviders();
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
i => i.Definition.Id,
s => s.ProviderId,
(i, s) => new { Indexer = i, Status = s })
(i, s) => new { Provider = i, Status = s })
.Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1))
.ToList();
if (backOffIndexers.Empty())
if (backOffProviders.Empty())
{
return new HealthCheck(GetType());
}
if (backOffIndexers.Count == enabledIndexers.Count)
if (backOffProviders.Count == enabledProviders.Count)
{
return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures");
}
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffIndexers.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures");
return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#indexers-are-unavailable-due-to-failures");
}
}
}

5
src/NzbDrone.Core/Indexers/IndexerBase.cs
View File

@ -96,11 +96,6 @@ namespace NzbDrone.Core.Indexers
failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message));
}
if (Definition.Id != 0)
{
_indexerStatusService.RecordSuccess(Definition.Id);
}
return new ValidationResult(failures);
}

19
src/NzbDrone.Core/Indexers/IndexerFactory.cs
View File

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.Linq;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Composition;
using NzbDrone.Core.Messaging.Events;
@ -21,7 +22,7 @@ namespace NzbDrone.Core.Indexers
public IndexerFactory(IIndexerStatusService indexerStatusService,
IIndexerRepository providerRepository,
IEnumerable<IIndexer> providers,
IContainer container,
IContainer container,
IEventAggregator eventAggregator,
Logger logger)
: base(providerRepository, providers, container, eventAggregator, logger)
@ -70,7 +71,7 @@ namespace NzbDrone.Core.Indexers
private IEnumerable<IIndexer> FilterBlockedIndexers(IEnumerable<IIndexer> indexers)
{
var blockedIndexers = _indexerStatusService.GetBlockedIndexers().ToDictionary(v => v.ProviderId, v => v);
var blockedIndexers = _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
foreach (var indexer in indexers)
{
@ -84,5 +85,17 @@ namespace NzbDrone.Core.Indexers
yield return indexer;
}
}
public override ValidationResult Test(IndexerDefinition definition)
{
var result = base.Test(definition);
if (result == null && definition.Id != 0)
{
_indexerStatusService.RecordSuccess(definition.Id);
}
return result;
}
}
}
}

19
src/NzbDrone.Core/Indexers/IndexerStatus.cs
View File

@ -1,23 +1,10 @@
using System;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.Indexers
{
public class IndexerStatus : ModelBase
public class IndexerStatus : ProviderStatusBase
{
public int ProviderId { get; set; }
public DateTime? InitialFailure { get; set; }
public DateTime? MostRecentFailure { get; set; }
public int EscalationLevel { get; set; }
public DateTime? DisabledTill { get; set; }
public ReleaseInfo LastRssSyncReleaseInfo { get; set; }
public bool IsDisabled()
{
return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow;
}
}
}

19
src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs
View File

@ -1,26 +1,19 @@
using System.Linq;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.Indexers
{
public interface IIndexerStatusRepository : IProviderRepository<IndexerStatus>
public interface IIndexerStatusRepository : IProviderStatusRepository<IndexerStatus>
{
IndexerStatus FindByIndexerId(int indexerId);
}
public class IndexerStatusRepository : ProviderRepository<IndexerStatus>, IIndexerStatusRepository
}
public class IndexerStatusRepository : ProviderStatusRepository<IndexerStatus>, IIndexerStatusRepository
{
public IndexerStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
public IndexerStatus FindByIndexerId(int indexerId)
{
return Query.Where(c => c.ProviderId == indexerId).SingleOrDefault();
}
}
}

126
src/NzbDrone.Core/Indexers/IndexerStatusService.cs
View File

@ -5,149 +5,39 @@ using NLog;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Events;
using NzbDrone.Core.ThingiProvider.Status;
namespace NzbDrone.Core.Indexers
{
public interface IIndexerStatusService
public interface IIndexerStatusService : IProviderStatusServiceBase<IndexerStatus>
{
List<IndexerStatus> GetBlockedIndexers();
ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId);
void RecordSuccess(int indexerId);
void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan));
void RecordConnectionFailure(int indexerId);
void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo);
}
public class IndexerStatusService : IIndexerStatusService, IHandleAsync<ProviderDeletedEvent<IIndexer>>
public class IndexerStatusService : ProviderStatusServiceBase<IIndexer, IndexerStatus>, IIndexerStatusService
{
private static readonly int[] EscalationBackOffPeriods = {
0,
5 * 60,
15 * 60,
30 * 60,
60 * 60,
3 * 60 * 60,
6 * 60 * 60,
12 * 60 * 60,
24 * 60 * 60
};
private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1;
private static readonly object _syncRoot = new object();
private readonly IIndexerStatusRepository _indexerStatusRepository;
private readonly Logger _logger;
public IndexerStatusService(IIndexerStatusRepository indexerStatusRepository, Logger logger)
public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, Logger logger)
: base(providerStatusRepository, logger)
{
_indexerStatusRepository = indexerStatusRepository;
_logger = logger;
}
public List<IndexerStatus> GetBlockedIndexers()
{
return _indexerStatusRepository.All().Where(v => v.IsDisabled()).ToList();
}
public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId)
{
return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo;
return GetProviderStatus(indexerId).LastRssSyncReleaseInfo;
}
private IndexerStatus GetIndexerStatus(int indexerId)
{
return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { ProviderId = indexerId };
}
private TimeSpan CalculateBackOffPeriod(IndexerStatus status)
{
var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
}
public void RecordSuccess(int indexerId)
{
lock (_syncRoot)
{
var status = GetIndexerStatus(indexerId);
if (status.EscalationLevel == 0)
{
return;
}
status.EscalationLevel--;
status.DisabledTill = null;
_indexerStatusRepository.Upsert(status);
}
}
protected void RecordFailure(int indexerId, TimeSpan minimumBackOff, bool escalate)
{
lock (_syncRoot)
{
var status = GetIndexerStatus(indexerId);
var now = DateTime.UtcNow;
if (status.EscalationLevel == 0)
{
status.InitialFailure = now;
}
status.MostRecentFailure = now;
if (escalate)
{
status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
}
if (minimumBackOff != TimeSpan.Zero)
{
while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff)
{
status.EscalationLevel++;
}
}
status.DisabledTill = now + CalculateBackOffPeriod(status);
_indexerStatusRepository.Upsert(status);
}
}
public void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan))
{
RecordFailure(indexerId, minimumBackOff, true);
}
public void RecordConnectionFailure(int indexerId)
{
RecordFailure(indexerId, default(TimeSpan), false);
}
public void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo)
{
lock (_syncRoot)
{
var status = GetIndexerStatus(indexerId);
var status = GetProviderStatus(indexerId);
status.LastRssSyncReleaseInfo = releaseInfo;
_indexerStatusRepository.Upsert(status);
}
}
public void HandleAsync(ProviderDeletedEvent<IIndexer> message)
{
var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId);
if (indexerStatus != null)
{
_indexerStatusRepository.Delete(indexerStatus);
_providerStatusRepository.Upsert(status);
}
}
}

3
src/NzbDrone.Core/NzbDrone.Core.csproj
View File

@ -1084,6 +1084,9 @@
<Compile Include="ThingiProvider\ProviderFactory.cs" />
<Compile Include="ThingiProvider\ProviderMessage.cs" />
<Compile Include="ThingiProvider\ProviderRepository.cs" />
<Compile Include="ThingiProvider\Status\ProviderStatusBase.cs" />
<Compile Include="ThingiProvider\Status\ProviderStatusRepository.cs" />
<Compile Include="ThingiProvider\Status\ProviderStatusServiceBase.cs" />
<Compile Include="TinyTwitter.cs" />
<Compile Include="Tv\Actor.cs" />
<Compile Include="Tv\AddSeriesOptions.cs" />

4
src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs
View File

@ -76,7 +76,7 @@ namespace NzbDrone.Core.ThingiProvider
return definitions;
}
public ValidationResult Test(TProviderDefinition definition)
public virtual ValidationResult Test(TProviderDefinition definition)
{
return GetInstance(definition).Test();
}
@ -168,4 +168,4 @@ namespace NzbDrone.Core.ThingiProvider
}
}
}
}
}

23
src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs Normal file
View File

@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.ThingiProvider.Status
{
public abstract class ProviderStatusBase : ModelBase
{
public int ProviderId { get; set; }
public DateTime? InitialFailure { get; set; }
public DateTime? MostRecentFailure { get; set; }
public int EscalationLevel { get; set; }
public DateTime? DisabledTill { get; set; }
public virtual bool IsDisabled()
{
return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow;
}
}
}

30
src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs Normal file
View File

@ -0,0 +1,30 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.ThingiProvider.Status
{
public interface IProviderStatusRepository<TModel> : IBasicRepository<TModel>
where TModel : ProviderStatusBase, new()
{
TModel FindByProviderId(int providerId);
}
public class ProviderStatusRepository<TModel> : BasicRepository<TModel>, IProviderStatusRepository<TModel>
where TModel : ProviderStatusBase, new()
{
public ProviderStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
: base(database, eventAggregator)
{
}
public TModel FindByProviderId(int providerId)
{
return Query.Where(c => c.ProviderId == providerId).SingleOrDefault();
}
}
}

137
src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs Normal file
View File

@ -0,0 +1,137 @@
using System;
using System.Collections.Generic;
using System.Linq;
using NLog;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Events;
namespace NzbDrone.Core.ThingiProvider.Status
{
public interface IProviderStatusServiceBase<TModel>
where TModel : ProviderStatusBase, new()
{
List<TModel> GetBlockedProviders();
void RecordSuccess(int providerId);
void RecordFailure(int providerId, TimeSpan minimumBackOff = default(TimeSpan));
void RecordConnectionFailure(int providerId);
}
public abstract class ProviderStatusServiceBase<TProvider, TModel> : IProviderStatusServiceBase<TModel>, IHandleAsync<ProviderDeletedEvent<TProvider>>
where TProvider : IProvider
where TModel : ProviderStatusBase, new()
{
private static readonly int[] EscalationBackOffPeriods = {
0,
5 * 60,
15 * 60,
30 * 60,
60 * 60,
3 * 60 * 60,
6 * 60 * 60,
12 * 60 * 60,
24 * 60 * 60
};
private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1;
protected readonly object _syncRoot = new object();
protected readonly IProviderStatusRepository<TModel> _providerStatusRepository;
protected readonly Logger _logger;
public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, Logger logger)
{
_providerStatusRepository = providerStatusRepository;
_logger = logger;
}
public virtual List<TModel> GetBlockedProviders()
{
return _providerStatusRepository.All().Where(v => v.IsDisabled()).ToList();
}
protected virtual TModel GetProviderStatus(int providerId)
{
return _providerStatusRepository.FindByProviderId(providerId) ?? new TModel { ProviderId = providerId };
}
protected virtual TimeSpan CalculateBackOffPeriod(TModel status)
{
var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel);
return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]);
}
public virtual void RecordSuccess(int providerId)
{
lock (_syncRoot)
{
var status = GetProviderStatus(providerId);
if (status.EscalationLevel == 0)
{
return;
}
status.EscalationLevel--;
status.DisabledTill = null;
_providerStatusRepository.Upsert(status);
}
}
protected virtual void RecordFailure(int providerId, TimeSpan minimumBackOff, bool escalate)
{
lock (_syncRoot)
{
var status = GetProviderStatus(providerId);
var now = DateTime.UtcNow;
if (status.EscalationLevel == 0)
{
status.InitialFailure = now;
}
status.MostRecentFailure = now;
if (escalate)
{
status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
}
if (minimumBackOff != TimeSpan.Zero)
{
while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff)
{
status.EscalationLevel++;
}
}
status.DisabledTill = now + CalculateBackOffPeriod(status);
_providerStatusRepository.Upsert(status);
}
}
public virtual void RecordFailure(int providerId, TimeSpan minimumBackOff = default(TimeSpan))
{
RecordFailure(providerId, minimumBackOff, true);
}
public virtual void RecordConnectionFailure(int providerId)
{
RecordFailure(providerId, default(TimeSpan), false);
}
public virtual void HandleAsync(ProviderDeletedEvent<TProvider> message)
{
var providerStatus = _providerStatusRepository.FindByProviderId(message.ProviderId);
if (providerStatus != null)
{
_providerStatusRepository.Delete(providerStatus);
}
}
}
}