From 12e5d1fad1978e8a5741bf97480d890b34e0cbd0 Mon Sep 17 00:00:00 2001 From: Michael Feinbier Date: Fri, 29 Sep 2023 17:14:47 +0200 Subject: [PATCH] Implement Remote File listing and local mapping --- .../PutioTests/PutioFixture.cs | 96 +++++++++++++------ .../PutioTests/PutioProxyFixtures.cs | 62 ++++++++++++ .../Download/Clients/Putio/Putio.cs | 70 +++++++++----- .../Download/Clients/Putio/PutioFile.cs | 20 ++++ .../Download/Clients/Putio/PutioProxy.cs | 27 ++++-- .../Download/Clients/Putio/PutioResponse.cs | 20 ++++ 6 files changed, 236 insertions(+), 59 deletions(-) create mode 100644 src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs index ef849b8c3..a90a7bf1c 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients.Putio; @@ -24,7 +25,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests [SetUp] public void Setup() { - _settings = new PutioSettings(); + _settings = new PutioSettings + { + SaveParentId = "1" + }; Subject.Definition = new DownloadClientDefinition(); Subject.Definition.Settings = _settings; @@ -72,7 +76,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests Size = 1000, Downloaded = 1000, SaveParentId = 1, - FileId = 1 + FileId = 2 }; _completed_different_parent = new PutioTorrent @@ -84,7 +88,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests Size = 1000, Downloaded = 1000, SaveParentId = 2, - FileId = 1 + FileId = 3 }; _seeding = new PutioTorrent @@ -97,7 +101,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests Downloaded = 1000, Uploaded = 1300, SaveParentId = 1, - FileId = 2 + FileId = 4 }; Mocker.GetMock() @@ -110,6 +114,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests Mocker.GetMock() .Setup(v => v.GetAccountSettings(It.IsAny())); + + Mocker.GetMock() + .Setup(v => v.GetFileListingResponse(It.IsAny(), It.IsAny())) + .Returns(PutioFileListingResponse.Empty()); } protected void GivenFailedDownload() @@ -121,18 +129,16 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests protected void GivenSuccessfulDownload() { - Mocker.GetMock() - .Setup(s => s.Get(It.IsAny())) - .Returns(r => new HttpResponse(r, new HttpHeader(), new byte[1000])); - /* - Mocker.GetMock() - .Setup(s => s.AddTorrentFromUrl(It.IsAny(), It.IsAny())) - .Callback(PrepareClientToReturnQueuedItem); + GivenRemoteFileStructure(new List + { + new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO }, + new PutioFile { Id = _seeding.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER }, + }, new PutioFile { Id = 1, Name = "Downloads" }); - Mocker.GetMock() - .Setup(s => s.AddTorrentFromData(It.IsAny(), It.IsAny())) - .Callback(PrepareClientToReturnQueuedItem); - */ + // GivenRemoteFileStructure(new List + // { + // new PutioFile { Id = _completed_different_parent.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO }, + // }, new PutioFile { Id = 2, Name = "Downloads_new" }); } protected virtual void GivenTorrents(List torrents) @@ -144,6 +150,20 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests .Returns(torrents); } + protected virtual void GivenRemoteFileStructure(List files, PutioFile parentFile) + { + files ??= new List(); + var list = new PutioFileListingResponse { Files = files, Parent = parentFile }; + + Mocker.GetMock() + .Setup(s => s.GetFileListingResponse(parentFile.Id, It.IsAny())) + .Returns(list); + + Mocker.GetMock() + .Setup(s => s.FolderExists(It.IsAny())) + .Returns(true); + } + protected virtual void GivenMetadata(List metadata) { metadata ??= new List(); @@ -170,12 +190,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests _seeding, _completed_different_parent }); - GivenMetadata(new List - { - PutioTorrentMetadata.fromTorrent(_completed, true), - PutioTorrentMetadata.fromTorrent(_seeding, true), - PutioTorrentMetadata.fromTorrent(_completed_different_parent, true), - }); + GivenSuccessfulDownload(); var items = Subject.GetItems(); @@ -184,9 +199,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests VerifyWarning(items.ElementAt(2)); VerifyCompleted(items.ElementAt(3)); VerifyCompleted(items.ElementAt(4)); - VerifyCompleted(items.ElementAt(5)); - items.Should().HaveCount(6); + items.Should().HaveCount(5); } [TestCase(1, 5)] @@ -203,6 +217,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests _seeding, _completed_different_parent }); + GivenSuccessfulDownload(); _settings.SaveParentId = configuredParentId.ToString(); @@ -225,7 +240,6 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests { _queued }); - GivenMetadata(new List { PutioTorrentMetadata.fromTorrent(_queued, true) }); var item = Subject.GetItems().Single(); @@ -233,13 +247,41 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests } [Test] - public void test_getItems_marks_non_existing_local_download_as_downloading() + public void test_getItems_path_for_folders() { GivenTorrents(new List { _completed }); - GivenMetadata(new List { PutioTorrentMetadata.fromTorrent(_completed, false) }); + GivenRemoteFileStructure(new List + { + new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER }, + }, new PutioFile { Id = 1, Name = "Downloads" }); var item = Subject.GetItems().Single(); - VerifyDownloading(item); + + VerifyCompleted(item); + item.OutputPath.ToString().Should().ContainAll("Downloads", _title); + + Mocker.GetMock() + .Verify(s => s.GetFileListingResponse(1, It.IsAny()), Times.AtLeastOnce()); + } + + [Test] + public void test_getItems_path_for_files() + { + GivenTorrents(new List { _completed }); + GivenRemoteFileStructure(new List + { + new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO }, + }, new PutioFile { Id = 1, Name = "Downloads" }); + + var item = Subject.GetItems().Single(); + + VerifyCompleted(item); + + item.OutputPath.ToString().Should().Contain("Downloads"); + item.OutputPath.ToString().Should().NotContain(_title); + + Mocker.GetMock() + .Verify(s => s.GetFileListingResponse(It.IsAny(), It.IsAny()), Times.AtLeastOnce()); } } } diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs index c945ff53b..12c8c6d15 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs @@ -59,6 +59,68 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests Assert.IsTrue(list["456"].Downloaded); } + [Test] + public void test_GetFileListingResponse() + { + var json = @"{ + ""cursor"": null, + ""files"": [ + { + ""file_type"": ""VIDEO"", + ""id"": 111, + ""name"": ""My.download.mkv"", + ""parent_id"": 4711 + }, + { + ""file_type"": ""FOLDER"", + ""id"": 222, + ""name"": ""Another-folder[dth]"", + ""parent_id"": 4711 + } + ], + ""parent"": { + ""file_type"": ""FOLDER"", + ""id"": 4711, + ""name"": ""Incoming"", + ""parent_id"": 0 + }, + ""status"": ""OK"", + ""total"": 2 + }"; + ClientGetWillReturn(json); + + var response = Subject.GetFileListingResponse(4711, new PutioSettings()); + + Assert.That(response, Is.Not.Null); + Assert.AreEqual(response.Files.Count, 2); + Assert.AreEqual(4711, response.Parent.Id); + Assert.AreEqual(111, response.Files[0].Id); + Assert.AreEqual(222, response.Files[1].Id); + } + + [Test] + public void test_GetFileListingResponse_empty() + { + var json = @"{ + ""cursor"": null, + ""files"": [], + ""parent"": { + ""file_type"": ""FOLDER"", + ""id"": 4711, + ""name"": ""Incoming"", + ""parent_id"": 0 + }, + ""status"": ""OK"", + ""total"": 0 + }"; + ClientGetWillReturn(json); + + var response = Subject.GetFileListingResponse(4711, new PutioSettings()); + + Assert.That(response, Is.Not.Null); + Assert.AreEqual(response.Files.Count, 0); + } + private void ClientGetWillReturn(string obj) where TResult : new() { diff --git a/src/NzbDrone.Core/Download/Clients/Putio/Putio.cs b/src/NzbDrone.Core/Download/Clients/Putio/Putio.cs index a385d68d4..d85ea023f 100644 --- a/src/NzbDrone.Core/Download/Clients/Putio/Putio.cs +++ b/src/NzbDrone.Core/Download/Clients/Putio/Putio.cs @@ -8,7 +8,6 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.MediaFiles.TorrentInfo; -using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using NzbDrone.Core.Validation; @@ -55,12 +54,20 @@ namespace NzbDrone.Core.Download.Clients.Putio public override IEnumerable GetItems() { List torrents; - Dictionary metadata; + PutioFileListingResponse fileListingResponse; try { torrents = _proxy.GetTorrents(Settings); - metadata = _proxy.GetAllTorrentMetadata(Settings); + + if (Settings.SaveParentId.IsNotNullOrWhiteSpace()) + { + fileListingResponse = _proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings); + } + else + { + fileListingResponse = _proxy.GetFileListingResponse(0, Settings); + } } catch (DownloadClientException ex) { @@ -100,29 +107,31 @@ namespace NzbDrone.Core.Download.Clients.Putio { if (torrent.FileId != 0) { - // How needs the output path need to look if we have remote files? + // Todo: make configurable? Behaviour might be different for users (rclone mount, vs sync/mv) + item.CanMoveFiles = false; + item.CanBeRemoved = false; - // check if we need to download the torrent from the remote - var title = FileNameBuilder.CleanFileName(torrent.Name); + var file = fileListingResponse.Files.FirstOrDefault(f => f.Id == torrent.FileId); + var parent = fileListingResponse.Parent; - // _diskProvider.FileExists(new OsPath()) - /* - var file = _proxy.GetFile(torrent.FileId, Settings); - var torrentPath = "/completed/" + file.Name; - - var outputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Url, new OsPath(torrentPath)); - - if (Settings.SaveParentId.IsNotNullOrWhiteSpace()) + if (file == null || parent == null) { - var directories = outputPath.FullPath.Split('\\', '/'); - if (!directories.Contains(string.Format("{0}", Settings.SaveParentId))) + item.Message = string.Format("Did not find file {0} in remote listing", torrent.FileId); + item.Status = DownloadItemStatus.Warning; + } + else + { + var expectedPath = new OsPath(Settings.DownloadPath) + new OsPath(parent.Name); + if (file.IsFolder()) { - continue; + expectedPath += new OsPath(file.Name); + } + + if (_diskProvider.FolderExists(expectedPath.FullPath)) + { + item.OutputPath = expectedPath; } } - - item.OutputPath = outputPath; // + torrent.Name; - */ } } catch (DownloadClientException ex) @@ -170,12 +179,9 @@ namespace NzbDrone.Core.Download.Clients.Putio public override DownloadClientInfo GetStatus() { - var destDir = new OsPath(Settings.DownloadPath); - return new DownloadClientInfo { - IsLocalhost = false, - OutputRootFolders = new List { destDir } + IsLocalhost = false }; } @@ -183,6 +189,7 @@ namespace NzbDrone.Core.Download.Clients.Putio { failures.AddIfNotNull(TestFolder(Settings.DownloadPath, "DownloadPath")); failures.AddIfNotNull(TestConnection()); + failures.AddIfNotNull(TestRemoteParentFolder()); if (failures.Any()) { return; @@ -229,6 +236,21 @@ namespace NzbDrone.Core.Download.Clients.Putio return null; } + private ValidationFailure TestRemoteParentFolder() + { + try + { + _proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings); + } + catch (Exception ex) + { + _logger.Error(ex, ex.Message); + return new NzbDroneValidationFailure("SaveParentId", "This is not a valid folder in your account"); + } + + return null; + } + public override void RemoveItem(DownloadClientItem item, bool deleteData) { throw new NotImplementedException(); diff --git a/src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs b/src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs new file mode 100644 index 000000000..973ed8095 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; + +namespace NzbDrone.Core.Download.Clients.Putio +{ + public class PutioFile + { + public static string FILE_TYPE_FOLDER = "FOLDER"; + public static string FILE_TYPE_VIDEO = "VIDEO"; + public long Id { get; set; } + public string Name { get; set; } + [JsonProperty(PropertyName = "parent_id")] + public long ParentId { get; set; } + [JsonProperty(PropertyName = "file_type")] + public string FileType { get; set; } + public bool IsFolder() + { + return FileType == FILE_TYPE_FOLDER; + } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs b/src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs index 902cc90f1..81bbb0216 100644 --- a/src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs @@ -13,10 +13,10 @@ namespace NzbDrone.Core.Download.Clients.Putio List GetTorrents(PutioSettings settings); void AddTorrentFromUrl(string torrentUrl, PutioSettings settings); void AddTorrentFromData(byte[] torrentData, PutioSettings settings); - void RemoveTorrent(string hash, PutioSettings settings); void GetAccountSettings(PutioSettings settings); public PutioTorrentMetadata GetTorrentMetadata(PutioTorrent torrent, PutioSettings settings); public Dictionary GetAllTorrentMetadata(PutioSettings settings); + public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings); } public class PutioProxy : IPutioProxy @@ -57,13 +57,6 @@ namespace NzbDrone.Core.Download.Clients.Putio // ProcessRequest(Method.POST, "transfers/add", arguments, settings); } - public void RemoveTorrent(string hashString, PutioSettings settings) - { - // var arguments = new Dictionary(); - // arguments.Add("transfer_ids", new string[] { hashString }); - // ProcessRequest(Method.POST, "torrents/cancel", arguments, settings); - } - public void GetAccountSettings(PutioSettings settings) { Execute(BuildRequest(HttpMethod.Get, "account/settings", settings)); @@ -85,6 +78,24 @@ namespace NzbDrone.Core.Download.Clients.Putio }; } + public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings) + { + var request = BuildRequest(HttpMethod.Get, "files/list", settings); + request.AddQueryParam("parent_id", parentId); + request.AddQueryParam("per_page", 1000); + + try + { + var response = Execute(request); + return response.Resource; + } + catch (DownloadClientException ex) + { + _logger.Error(ex, "Failed to get file listing response"); + throw; + } + } + public Dictionary GetAllTorrentMetadata(PutioSettings settings) { var metadata = Execute(BuildRequest(HttpMethod.Get, "config", settings)); diff --git a/src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs b/src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs index 4b33eec93..e47bd2a33 100644 --- a/src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs +++ b/src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs @@ -25,4 +25,24 @@ namespace NzbDrone.Core.Download.Clients.Putio { public Dictionary Config { get; set; } } + + public class PutioFileListingResponse : PutioGenericResponse + { + public static PutioFileListingResponse Empty() + { + return new PutioFileListingResponse + { + Files = new List(), + Parent = new PutioFile + { + Id = 0, + Name = "Your Files" + } + }; + } + + public List Files { get; set; } + + public PutioFile Parent { get; set; } + } }