Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated migration to attempt associate old grabbed & imported events and associate drone factory imports during CompletedDownloadHandling.

This commit is contained in:
Taloth Saldono 2014-05-29 00:48:37 +02:00
parent 1b96a43037
commit 1a63b1caba
8 changed files with 331 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs
View File

@ -13,6 +13,8 @@ using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Test.Common;
using NzbDrone.Core.Tv;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Test.Download
{
@ -28,6 +30,10 @@ namespace NzbDrone.Core.Test.Download
.All()
.With(h => h.Status = DownloadItemStatus.Completed)
.With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic())
.With(h => h.RemoteEpisode = new RemoteEpisode
{
Episodes = new List<Episode> { new Episode { Id = 1 } }
})
.Build()
.ToList();
@ -255,7 +261,9 @@ namespace NzbDrone.Core.Test.Download
history.First().Data.Add("downloadClient", "SabnzbdClient");
history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId);
Subject.Execute(new CheckForFinishedDownloadCommand());
VerifyNoImports();
}
@ -278,9 +286,60 @@ namespace NzbDrone.Core.Test.Download
history.First().Data.Add("downloadClient", "SabnzbdClient");
history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId);
Subject.Execute(new CheckForFinishedDownloadCommand());
VerifyNoImports();
}
[Test]
public void should_process_as_already_imported_if_drone_factory_import_history_exists()
{
GivenCompletedDownloadClientHistory(false);
_completed.Clear();
_completed.AddRange(Builder<DownloadClientItem>.CreateListOfSize(2)
.All()
.With(h => h.Status = DownloadItemStatus.Completed)
.With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic())
.With(h => h.RemoteEpisode = new RemoteEpisode
{
Episodes = new List<Episode> { new Episode { Id = 1 } }
})
.Build());
var grabbedHistory = Builder<History.History>.CreateListOfSize(2)
.All()
.With(d => d.Data["downloadClient"] = "SabnzbdClient")
.TheFirst(1)
.With(d => d.Data["downloadClientId"] = _completed.First().DownloadClientId)
.With(d => d.SourceTitle = "Droned.S01E01.720p-LAZY")
.TheLast(1)
.With(d => d.Data["downloadClientId"] = _completed.Last().DownloadClientId)
.With(d => d.SourceTitle = "Droned.S01E01.Proper.720p-LAZY")
.Build()
.ToList();
var importedHistory = Builder<History.History>.CreateListOfSize(2)
.All()
.With(d => d.EpisodeId = 1)
.TheFirst(1)
.With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic())
.TheLast(1)
.With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.Proper.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic())
.Build()
.ToList();
GivenGrabbedHistory(grabbedHistory);
GivenImportedHistory(importedHistory);
Subject.Execute(new CheckForFinishedDownloadCommand());
VerifyNoImports();
Mocker.GetMock<IHistoryService>()
.Verify(v => v.UpdateHistoryData(It.IsAny<int>(), It.IsAny<Dictionary<String, String>>()), Times.Exactly(2));
}
[Test]
public void should_not_remove_if_config_disabled()
{

224
src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs Normal file
View File

@ -0,0 +1,224 @@
using System;
using System.Data;
using System.Linq;
using System.Collections.Generic;
using FluentMigrator;
using Newtonsoft.Json;
using NzbDrone.Common;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Datastore.Migration.Framework;
using System.IO;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(51)]
public class download_client_import : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Execute.WithConnection(ConvertFolderSettings);
Execute.WithConnection(AssociateImportedHistoryItems);
}
private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran)
{
using (IDbCommand downloadClientsCmd = conn.CreateCommand())
{
downloadClientsCmd.Transaction = tran;
downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'";
var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String;
downloadClientsCmd.Transaction = tran;
downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'";
using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader())
{
while (downloadClientReader.Read())
{
var id = downloadClientReader.GetInt32(0);
var implementation = downloadClientReader.GetString(1);
var settings = downloadClientReader.GetString(2);
var configContract = downloadClientReader.GetString(3);
var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject;
if (implementation == "Blackhole")
{
var newSettings = new
{
NzbFolder = settingsJson.Value<String>("folder"),
WatchFolder = downloadedEpisodesFolder
}.ToJson();
using (IDbCommand updateCmd = conn.CreateCommand())
{
updateCmd.Transaction = tran;
updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?";
updateCmd.AddParameter("UsenetBlackhole");
updateCmd.AddParameter(newSettings);
updateCmd.AddParameter("UsenetBlackholeSettings");
updateCmd.AddParameter(id);
updateCmd.ExecuteNonQuery();
}
}
else if (implementation == "Pneumatic")
{
var newSettings = new
{
NzbFolder = settingsJson.Value<String>("folder")
}.ToJson();
using (IDbCommand updateCmd = conn.CreateCommand())
{
updateCmd.Transaction = tran;
updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?";
updateCmd.AddParameter(newSettings);
updateCmd.AddParameter("PneumaticSettings");
updateCmd.AddParameter(id);
updateCmd.ExecuteNonQuery();
}
}
else
{
using (IDbCommand updateCmd = conn.CreateCommand())
{
updateCmd.Transaction = tran;
updateCmd.CommandText = "DELETE FROM DownloadClients WHERE Id = ?";
updateCmd.AddParameter(id);
updateCmd.ExecuteNonQuery();
}
}
}
}
}
}
private sealed class MigrationHistoryItem
{
public Int32 Id { get; set; }
public Int32 EpisodeId { get; set; }
public Int32 SeriesId { get; set; }
public String SourceTitle { get; set; }
public DateTime Date { get; set; }
public Dictionary<String, String> Data { get; set; }
public MigrationHistoryEventType EventType { get; set; }
}
private enum MigrationHistoryEventType
{
Unknown = 0,
Grabbed = 1,
SeriesFolderImported = 2,
DownloadFolderImported = 3,
DownloadFailed = 4
}
private void AssociateImportedHistoryItems(IDbConnection conn, IDbTransaction tran)
{
var historyItems = new List<MigrationHistoryItem>();
using (IDbCommand historyCmd = conn.CreateCommand())
{
historyCmd.Transaction = tran;
historyCmd.CommandText = @"SELECT Id, EpisodeId, SeriesId, SourceTitle, Date, Data, EventType FROM History WHERE EventType NOT NULL";
using (IDataReader historyRead = historyCmd.ExecuteReader())
{
while (historyRead.Read())
{
historyItems.Add(new MigrationHistoryItem
{
Id = historyRead.GetInt32(0),
EpisodeId = historyRead.GetInt32(1),
SeriesId = historyRead.GetInt32(2),
SourceTitle = historyRead.GetString(3),
Date = historyRead.GetDateTime(4),
Data = Json.Deserialize<Dictionary<String, String>>(historyRead.GetString(5)),
EventType = (MigrationHistoryEventType)historyRead.GetInt32(6)
});
}
}
}
var numHistoryItemsNotAssociated = historyItems.Count(v => v.EventType == MigrationHistoryEventType.DownloadFolderImported &&
v.Data.GetValueOrDefault("downloadClientId") == null);
if (numHistoryItemsNotAssociated == 0)
{
return;
}
var historyItemsToAssociate = new Dictionary<MigrationHistoryItem, MigrationHistoryItem>();
var historyItemsLookup = historyItems.ToLookup(v => v.EpisodeId);
foreach (var historyItemGroup in historyItemsLookup)
{
var list = historyItemGroup.ToList();
for (int i = 0; i < list.Count - 1; i++)
{
var grabbedEvent = list[i];
if (grabbedEvent.EventType != MigrationHistoryEventType.Grabbed) continue;
if (grabbedEvent.Data.GetValueOrDefault("downloadClient") == null || grabbedEvent.Data.GetValueOrDefault("downloadClientId") == null) continue;
// Check if it is already associated with a failed/imported event.
int j;
for (j = i + 1; j < list.Count;j++)
{
if (list[j].EventType != MigrationHistoryEventType.DownloadFolderImported &&
list[j].EventType != MigrationHistoryEventType.DownloadFailed)
{
continue;
}
if (list[j].Data.ContainsKey("downloadClient") && list[j].Data["downloadClient"] == grabbedEvent.Data["downloadClient"] &&
list[j].Data.ContainsKey("downloadClientId") && list[j].Data["downloadClientId"] == grabbedEvent.Data["downloadClientId"])
{
break;
}
}
if (j != list.Count)
{
list.RemoveAt(j);
list.RemoveAt(i--);
continue;
}
var importedEvent = list[i + 1];
if (importedEvent.EventType != MigrationHistoryEventType.DownloadFolderImported) continue;
var droppedPath = importedEvent.Data.GetValueOrDefault("droppedPath");
if (droppedPath != null && new FileInfo(droppedPath).Directory.Name == grabbedEvent.SourceTitle)
{
historyItemsToAssociate[importedEvent] = grabbedEvent;
list.RemoveAt(i + 1);
list.RemoveAt(i--);
}
}
}
foreach (var pair in historyItemsToAssociate)
{
using (IDbCommand updateHistoryCmd = conn.CreateCommand())
{
pair.Key.Data["downloadClient"] = pair.Value.Data["downloadClient"];
pair.Key.Data["downloadClientId"] = pair.Value.Data["downloadClientId"];
updateHistoryCmd.Transaction = tran;
updateHistoryCmd.CommandText = "UPDATE History SET Data = ? WHERE Id = ?";
updateHistoryCmd.AddParameter(pair.Key.Data.ToJson());
updateHistoryCmd.AddParameter(pair.Key.Id);
updateHistoryCmd.ExecuteNonQuery();
}
}
_logger.Info("Updated old History items. {0}/{1} old ImportedEvents were associated with GrabbedEvents.", historyItemsToAssociate.Count, numHistoryItemsNotAssociated);
}
}
}

88
src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs
View File

@ -1,88 +0,0 @@
using NzbDrone.Core.Datastore.Migration.Framework;
using FluentMigrator;
using System.Data;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Download.Clients.UsenetBlackhole;
using Newtonsoft.Json;
using System;
using NzbDrone.Core.Download.Clients.Pneumatic;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(51)]
public class rename_download_client_settings : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Execute.WithConnection(ConvertFolderSettings);
}
private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran)
{
using (IDbCommand downloadClientsCmd = conn.CreateCommand())
{
downloadClientsCmd.Transaction = tran;
downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'";
var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String;
downloadClientsCmd.Transaction = tran;
downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'";
using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader())
{
while (downloadClientReader.Read())
{
var id = downloadClientReader.GetInt32(0);
var implementation = downloadClientReader.GetString(1);
var settings = downloadClientReader.GetString(2);
var configContract = downloadClientReader.GetString(3);
var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject;
if (implementation == "Blackhole")
{
var newSettings = new
{
NzbFolder = settingsJson.Value<String>("folder"),
WatchFolder = downloadedEpisodesFolder
}.ToJson();
using (IDbCommand updateCmd = conn.CreateCommand())
{
updateCmd.Transaction = tran;
updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?";
updateCmd.AddParameter("UsenetBlackhole");
updateCmd.AddParameter(newSettings);
updateCmd.AddParameter("UsenetBlackholeSettings");
updateCmd.AddParameter(id);
updateCmd.ExecuteNonQuery();
}
}
else if (implementation == "Pneumatic")
{
var newSettings = new
{
NzbFolder = settingsJson.Value<String>("folder")
}.ToJson();
using (IDbCommand updateCmd = conn.CreateCommand())
{
updateCmd.Transaction = tran;
updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?";
updateCmd.AddParameter(newSettings);
updateCmd.AddParameter("PneumaticSettings");
updateCmd.AddParameter(id);
updateCmd.ExecuteNonQuery();
}
}
else
{
throw new NotSupportedException();
}
}
}
}
}
}
}

4
src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs
View File

@ -6,11 +6,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
{
public abstract class NzbDroneMigrationBase : FluentMigrator.Migration
{
private Logger _logger;
protected readonly Logger _logger;
protected NzbDroneMigrationBase()
{
_logger = NzbDroneLogger.GetLogger();
_logger = NzbDroneLogger.GetLogger(this);
}
protected virtual void MainDbUpgrade()

33
src/NzbDrone.Core/Download/CompletedDownloadService.cs
View File

@ -30,18 +30,21 @@ namespace NzbDrone.Core.Download
private readonly IConfigService _configService;
private readonly IDiskProvider _diskProvider;
private readonly IDownloadedEpisodesImportService _downloadedEpisodesImportService;
private readonly IHistoryService _historyService;
private readonly Logger _logger;
public CompletedDownloadService(IEventAggregator eventAggregator,
IConfigService configService,
IDiskProvider diskProvider,
IDownloadedEpisodesImportService downloadedEpisodesImportService,
IHistoryService historyService,
Logger logger)
{
_eventAggregator = eventAggregator;
_configService = configService;
_diskProvider = diskProvider;
_downloadedEpisodesImportService = downloadedEpisodesImportService;
_historyService = historyService;
_logger = logger;
}
@ -67,7 +70,7 @@ namespace NzbDrone.Core.Download
_logger.Trace("Ignoring download that wasn't grabbed by drone: " + trackedDownload.DownloadItem.Title);
return;
}
var importedItems = GetHistoryItems(importedHistory, trackedDownload.DownloadItem.DownloadClientId);
if (importedItems.Any())
@ -112,6 +115,34 @@ namespace NzbDrone.Core.Download
}
else
{
if (grabbedItems.Any())
{
var episodeIds = trackedDownload.DownloadItem.RemoteEpisode.Episodes.Select(v => v.Id).ToList();
// Check if we can associate it with a previous drone factory import.
importedItems = importedHistory.Where(v => v.Data.GetValueOrDefault(DownloadTrackingService.DOWNLOAD_CLIENT_ID) == null &&
episodeIds.Contains(v.EpisodeId) &&
v.Data.GetValueOrDefault("droppedPath") != null &&
new FileInfo(v.Data["droppedPath"]).Directory.Name == grabbedItems.First().SourceTitle
).ToList();
if (importedItems.Count == 1)
{
var importedFile = new FileInfo(importedItems.First().Data["droppedPath"]);
if (importedFile.Directory.Name == grabbedItems.First().SourceTitle)
{
trackedDownload.State = TrackedDownloadState.Imported;
importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT];
importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID];
_historyService.UpdateHistoryData(importedItems.First().Id, importedItems.First().Data);
_logger.Debug("Storage path does not exist, but found probable drone factory ImportEvent: " + trackedDownload.DownloadItem.Title);
return;
}
}
}
_logger.Debug("Storage path does not exist: " + trackedDownload.DownloadItem.Title);
return;
}

8
src/NzbDrone.Core/History/HistoryService.cs
View File

@ -25,6 +25,7 @@ namespace NzbDrone.Core.History
History MostRecentForEpisode(int episodeId);
History Get(int id);
List<History> FindBySourceTitle(string sourceTitle);
void UpdateHistoryData(Int32 historyId, Dictionary<String, String> data);
}
public class HistoryService : IHistoryService, IHandle<EpisodeGrabbedEvent>, IHandle<EpisodeImportedEvent>, IHandle<DownloadFailedEvent>
@ -101,6 +102,13 @@ namespace NzbDrone.Core.History
.FirstOrDefault();
}
public void UpdateHistoryData(Int32 historyId, Dictionary<String, String> data)
{
var history = _historyRepository.Get(historyId);
history.Data = data;
_historyRepository.Update(history);
}
public void Handle(EpisodeGrabbedEvent message)
{
foreach (var episode in message.Episode.Episodes)

12
src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs
View File

@ -160,7 +160,8 @@ namespace NzbDrone.Core.MediaFiles
}
}
return ProcessFiles(series, quality, videoFiles);
var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality);
return _importApprovedEpisodes.Import(decisions, true);
}
private void ProcessVideoFile(string videoFile)
@ -179,13 +180,8 @@ namespace NzbDrone.Core.MediaFiles
return;
}
ProcessFiles(series, null, videoFile);
}
private List<ImportDecision> ProcessFiles(Series series, QualityModel quality, params string[] videoFiles)
{
var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality);
return _importApprovedEpisodes.Import(decisions, true);
var decisions = _importDecisionMaker.GetImportDecisions(new [] { videoFile }.ToList(), series, true, null);
_importApprovedEpisodes.Import(decisions, true);
}
private void ProcessFolder(string path)

2
src/NzbDrone.Core/NzbDrone.Core.csproj
View File

@ -197,7 +197,7 @@
<Compile Include="Datastore\Migration\048_add_title_to_scenemappings.cs" />
<Compile Include="Datastore\Migration\049_fix_dognzb_url.cs" />
<Compile Include="Datastore\Migration\050_add_hash_to_metadata_files.cs" />
<Compile Include="Datastore\Migration\051_rename_download_client_settings.cs" />
<Compile Include="Datastore\Migration\051_download_client_import.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
<Compile Include="Datastore\Migration\Framework\MigrationExtension.cs" />