Implement Remote File listing and local mapping

src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.Disk;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Download.Clients.Putio;
@@ -24,7 +25,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
         [SetUp]
         public void Setup()
         {
-            _settings = new PutioSettings();
+            _settings = new PutioSettings
+            {
+                SaveParentId = "1"
+            };
 
             Subject.Definition = new DownloadClientDefinition();
             Subject.Definition.Settings = _settings;
@@ -72,7 +76,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                     Size = 1000,
                     Downloaded = 1000,
                     SaveParentId = 1,
-                    FileId = 1
+                    FileId = 2
                 };
 
             _completed_different_parent = new PutioTorrent
@@ -84,7 +88,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                     Size = 1000,
                     Downloaded = 1000,
                     SaveParentId = 2,
-                    FileId = 1
+                    FileId = 3
                 };
 
             _seeding = new PutioTorrent
@@ -97,7 +101,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                     Downloaded = 1000,
                     Uploaded = 1300,
                     SaveParentId = 1,
-                    FileId = 2
+                    FileId = 4
                 };
 
             Mocker.GetMock<ITorrentFileInfoReader>()
@@ -110,6 +114,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
 
             Mocker.GetMock<IPutioProxy>()
                 .Setup(v => v.GetAccountSettings(It.IsAny<PutioSettings>()));
+
+            Mocker.GetMock<IPutioProxy>()
+                .Setup(v => v.GetFileListingResponse(It.IsAny<long>(), It.IsAny<PutioSettings>()))
+                .Returns(PutioFileListingResponse.Empty());
         }
 
         protected void GivenFailedDownload()
@@ -121,18 +129,16 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
 
         protected void GivenSuccessfulDownload()
         {
-            Mocker.GetMock<IHttpClient>()
+            GivenRemoteFileStructure(new List<PutioFile>
-                  .Setup(s => s.Get(It.IsAny<HttpRequest>()))
+            {
-                  .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), new byte[1000]));
+                new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
-            /*
+                new PutioFile { Id = _seeding.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER },
-            Mocker.GetMock<IPutioProxy>()
+            }, new PutioFile { Id = 1, Name = "Downloads" });
-                .Setup(s => s.AddTorrentFromUrl(It.IsAny<string>(), It.IsAny<PutioSettings>()))
-                .Callback(PrepareClientToReturnQueuedItem);
 
-            Mocker.GetMock<IPutioProxy>()
+            // GivenRemoteFileStructure(new List<PutioFile>
-                .Setup(s => s.AddTorrentFromData(It.IsAny<byte[]>(), It.IsAny<PutioSettings>()))
+            // {
-                .Callback(PrepareClientToReturnQueuedItem);
+            //     new PutioFile { Id = _completed_different_parent.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
-            */
+            // }, new PutioFile { Id = 2, Name = "Downloads_new" });
         }
 
         protected virtual void GivenTorrents(List<PutioTorrent> torrents)
@@ -144,6 +150,20 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                 .Returns(torrents);
         }
 
+        protected virtual void GivenRemoteFileStructure(List<PutioFile> files, PutioFile parentFile)
+        {
+            files ??= new List<PutioFile>();
+            var list = new PutioFileListingResponse { Files = files, Parent = parentFile };
+
+            Mocker.GetMock<IPutioProxy>()
+                .Setup(s => s.GetFileListingResponse(parentFile.Id, It.IsAny<PutioSettings>()))
+                .Returns(list);
+
+            Mocker.GetMock<IDiskProvider>()
+                .Setup(s => s.FolderExists(It.IsAny<string>()))
+                .Returns(true);
+        }
+
         protected virtual void GivenMetadata(List<PutioTorrentMetadata> metadata)
         {
             metadata ??= new List<PutioTorrentMetadata>();
@@ -170,12 +190,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                 _seeding,
                 _completed_different_parent
             });
-            GivenMetadata(new List<PutioTorrentMetadata>
+            GivenSuccessfulDownload();
-            {
-                PutioTorrentMetadata.fromTorrent(_completed, true),
-                PutioTorrentMetadata.fromTorrent(_seeding, true),
-                PutioTorrentMetadata.fromTorrent(_completed_different_parent, true),
-            });
 
             var items = Subject.GetItems();
 
@@ -184,9 +199,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
             VerifyWarning(items.ElementAt(2));
             VerifyCompleted(items.ElementAt(3));
             VerifyCompleted(items.ElementAt(4));
-            VerifyCompleted(items.ElementAt(5));
 
-            items.Should().HaveCount(6);
+            items.Should().HaveCount(5);
         }
 
         [TestCase(1, 5)]
@@ -203,6 +217,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
                     _seeding,
                     _completed_different_parent
             });
+            GivenSuccessfulDownload();
 
             _settings.SaveParentId = configuredParentId.ToString();
 
@@ -225,7 +240,6 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
             {
                     _queued
             });
-            GivenMetadata(new List<PutioTorrentMetadata> { PutioTorrentMetadata.fromTorrent(_queued, true) });
 
             var item = Subject.GetItems().Single();
 
@@ -233,13 +247,41 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
         }
 
         [Test]
-        public void test_getItems_marks_non_existing_local_download_as_downloading()
+        public void test_getItems_path_for_folders()
         {
             GivenTorrents(new List<PutioTorrent> { _completed });
-            GivenMetadata(new List<PutioTorrentMetadata> { PutioTorrentMetadata.fromTorrent(_completed, false) });
+            GivenRemoteFileStructure(new List<PutioFile>
+            {
+                new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER },
+            }, new PutioFile { Id = 1, Name = "Downloads" });
 
             var item = Subject.GetItems().Single();
-            VerifyDownloading(item);
+
+            VerifyCompleted(item);
+            item.OutputPath.ToString().Should().ContainAll("Downloads", _title);
+
+            Mocker.GetMock<IPutioProxy>()
+                .Verify(s => s.GetFileListingResponse(1, It.IsAny<PutioSettings>()), Times.AtLeastOnce());
+        }
+
+        [Test]
+        public void test_getItems_path_for_files()
+        {
+            GivenTorrents(new List<PutioTorrent> { _completed });
+            GivenRemoteFileStructure(new List<PutioFile>
+            {
+                new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
+            }, new PutioFile { Id = 1, Name = "Downloads" });
+
+            var item = Subject.GetItems().Single();
+
+            VerifyCompleted(item);
+
+            item.OutputPath.ToString().Should().Contain("Downloads");
+            item.OutputPath.ToString().Should().NotContain(_title);
+
+            Mocker.GetMock<IPutioProxy>()
+            .Verify(s => s.GetFileListingResponse(It.IsAny<long>(), It.IsAny<PutioSettings>()), Times.AtLeastOnce());
         }
     }
 }

src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs

@@ -59,6 +59,68 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
             Assert.IsTrue(list["456"].Downloaded);
         }
 
+        [Test]
+        public void test_GetFileListingResponse()
+        {
+            var json = @"{
+                ""cursor"": null,
+                ""files"": [
+                    {
+                        ""file_type"": ""VIDEO"",
+                        ""id"": 111,
+                        ""name"": ""My.download.mkv"",
+                        ""parent_id"": 4711
+                    },
+                    {
+                        ""file_type"": ""FOLDER"",
+                        ""id"": 222,
+                        ""name"": ""Another-folder[dth]"",
+                        ""parent_id"": 4711
+                    }
+                ],
+                ""parent"": {
+                    ""file_type"": ""FOLDER"",
+                    ""id"": 4711,
+                    ""name"": ""Incoming"",
+                    ""parent_id"": 0
+                },
+                ""status"": ""OK"",
+                ""total"": 2
+            }";
+            ClientGetWillReturn<PutioFileListingResponse>(json);
+
+            var response = Subject.GetFileListingResponse(4711, new PutioSettings());
+
+            Assert.That(response, Is.Not.Null);
+            Assert.AreEqual(response.Files.Count, 2);
+            Assert.AreEqual(4711, response.Parent.Id);
+            Assert.AreEqual(111, response.Files[0].Id);
+            Assert.AreEqual(222, response.Files[1].Id);
+        }
+
+        [Test]
+        public void test_GetFileListingResponse_empty()
+        {
+            var json = @"{
+                ""cursor"": null,
+                ""files"": [],
+                ""parent"": {
+                    ""file_type"": ""FOLDER"",
+                    ""id"": 4711,
+                    ""name"": ""Incoming"",
+                    ""parent_id"": 0
+                },
+                ""status"": ""OK"",
+                ""total"": 0
+            }";
+            ClientGetWillReturn<PutioFileListingResponse>(json);
+
+            var response = Subject.GetFileListingResponse(4711, new PutioSettings());
+
+            Assert.That(response, Is.Not.Null);
+            Assert.AreEqual(response.Files.Count, 0);
+        }
+
         private void ClientGetWillReturn<TResult>(string obj)
             where TResult : new()
         {

src/NzbDrone.Core/Download/Clients/Putio/Putio.cs

@@ -8,7 +8,6 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.MediaFiles.TorrentInfo;
-using NzbDrone.Core.Organizer;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
 using NzbDrone.Core.Validation;
@@ -55,12 +54,20 @@ namespace NzbDrone.Core.Download.Clients.Putio
         public override IEnumerable<DownloadClientItem> GetItems()
         {
             List<PutioTorrent> torrents;
-            Dictionary<string, PutioTorrentMetadata> metadata;
+            PutioFileListingResponse fileListingResponse;
 
             try
             {
                 torrents = _proxy.GetTorrents(Settings);
-                metadata = _proxy.GetAllTorrentMetadata(Settings);
+
+                if (Settings.SaveParentId.IsNotNullOrWhiteSpace())
+                {
+                    fileListingResponse = _proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings);
+                }
+                else
+                {
+                    fileListingResponse = _proxy.GetFileListingResponse(0, Settings);
+                }
             }
             catch (DownloadClientException ex)
             {
@@ -100,29 +107,31 @@ namespace NzbDrone.Core.Download.Clients.Putio
                 {
                     if (torrent.FileId != 0)
                     {
-                        // How needs the output path need to look if we have remote files?
+                        // Todo: make configurable? Behaviour might be different for users (rclone mount, vs sync/mv)
+                        item.CanMoveFiles = false;
+                        item.CanBeRemoved = false;
 
-                        // check if we need to download the torrent from the remote
+                        var file = fileListingResponse.Files.FirstOrDefault(f => f.Id == torrent.FileId);
-                        var title = FileNameBuilder.CleanFileName(torrent.Name);
+                        var parent = fileListingResponse.Parent;
 
-                        // _diskProvider.FileExists(new OsPath())
+                        if (file == null || parent == null)
-                        /*
-                        var file = _proxy.GetFile(torrent.FileId, Settings);
-                        var torrentPath = "/completed/" + file.Name;
-
-                        var outputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Url, new OsPath(torrentPath));
-
-                        if (Settings.SaveParentId.IsNotNullOrWhiteSpace())
                         {
-                            var directories = outputPath.FullPath.Split('\\', '/');
+                            item.Message = string.Format("Did not find file {0} in remote listing", torrent.FileId);
-                            if (!directories.Contains(string.Format("{0}", Settings.SaveParentId)))
+                            item.Status = DownloadItemStatus.Warning;
+                        }
+                        else
+                        {
+                            var expectedPath = new OsPath(Settings.DownloadPath) + new OsPath(parent.Name);
+                            if (file.IsFolder())
                             {
-                                continue;
+                                expectedPath += new OsPath(file.Name);
+                            }
+
+                            if (_diskProvider.FolderExists(expectedPath.FullPath))
+                            {
+                                item.OutputPath = expectedPath;
                             }
                         }
-
-                        item.OutputPath = outputPath; // + torrent.Name;
-                        */
                     }
                 }
                 catch (DownloadClientException ex)
@@ -170,12 +179,9 @@ namespace NzbDrone.Core.Download.Clients.Putio
 
         public override DownloadClientInfo GetStatus()
         {
-            var destDir = new OsPath(Settings.DownloadPath);
-
             return new DownloadClientInfo
             {
-                IsLocalhost = false,
+                IsLocalhost = false
-                OutputRootFolders = new List<OsPath> { destDir }
             };
         }
 
@@ -183,6 +189,7 @@ namespace NzbDrone.Core.Download.Clients.Putio
         {
             failures.AddIfNotNull(TestFolder(Settings.DownloadPath, "DownloadPath"));
             failures.AddIfNotNull(TestConnection());
+            failures.AddIfNotNull(TestRemoteParentFolder());
             if (failures.Any())
             {
                 return;
@@ -229,6 +236,21 @@ namespace NzbDrone.Core.Download.Clients.Putio
             return null;
         }
 
+        private ValidationFailure TestRemoteParentFolder()
+        {
+            try
+            {
+                _proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings);
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, ex.Message);
+                return new NzbDroneValidationFailure("SaveParentId", "This is not a valid folder in your account");
+            }
+
+            return null;
+        }
+
         public override void RemoveItem(DownloadClientItem item, bool deleteData)
         {
             throw new NotImplementedException();

src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs

@@ -0,0 +1,20 @@
+using Newtonsoft.Json;
+
+namespace NzbDrone.Core.Download.Clients.Putio
+{
+    public class PutioFile
+    {
+        public static string FILE_TYPE_FOLDER = "FOLDER";
+        public static string FILE_TYPE_VIDEO = "VIDEO";
+        public long Id { get; set; }
+        public string Name { get; set; }
+        [JsonProperty(PropertyName = "parent_id")]
+        public long ParentId { get; set; }
+        [JsonProperty(PropertyName = "file_type")]
+        public string FileType { get; set; }
+        public bool IsFolder()
+        {
+            return FileType == FILE_TYPE_FOLDER;
+        }
+    }
+}

src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs

@@ -13,10 +13,10 @@ namespace NzbDrone.Core.Download.Clients.Putio
         List<PutioTorrent> GetTorrents(PutioSettings settings);
         void AddTorrentFromUrl(string torrentUrl, PutioSettings settings);
         void AddTorrentFromData(byte[] torrentData, PutioSettings settings);
-        void RemoveTorrent(string hash, PutioSettings settings);
         void GetAccountSettings(PutioSettings settings);
         public PutioTorrentMetadata GetTorrentMetadata(PutioTorrent torrent, PutioSettings settings);
         public Dictionary<string, PutioTorrentMetadata> GetAllTorrentMetadata(PutioSettings settings);
+        public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings);
     }
 
     public class PutioProxy : IPutioProxy
@@ -57,13 +57,6 @@ namespace NzbDrone.Core.Download.Clients.Putio
             // ProcessRequest<PutioGenericResponse>(Method.POST, "transfers/add", arguments, settings);
         }
 
-        public void RemoveTorrent(string hashString, PutioSettings settings)
-        {
-            // var arguments = new Dictionary<string, object>();
-            // arguments.Add("transfer_ids", new string[] { hashString });
-            // ProcessRequest<PutioGenericResponse>(Method.POST, "torrents/cancel", arguments, settings);
-        }
-
         public void GetAccountSettings(PutioSettings settings)
         {
             Execute<PutioGenericResponse>(BuildRequest(HttpMethod.Get, "account/settings", settings));
@@ -85,6 +78,24 @@ namespace NzbDrone.Core.Download.Clients.Putio
             };
         }
 
+        public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings)
+        {
+            var request = BuildRequest(HttpMethod.Get, "files/list", settings);
+            request.AddQueryParam("parent_id", parentId);
+            request.AddQueryParam("per_page", 1000);
+
+            try
+            {
+                var response = Execute<PutioFileListingResponse>(request);
+                return response.Resource;
+            }
+            catch (DownloadClientException ex)
+            {
+                _logger.Error(ex, "Failed to get file listing response");
+                throw;
+            }
+        }
+
         public Dictionary<string, PutioTorrentMetadata> GetAllTorrentMetadata(PutioSettings settings)
         {
             var metadata = Execute<PutioAllConfigResponse>(BuildRequest(HttpMethod.Get, "config", settings));

src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs

@@ -25,4 +25,24 @@ namespace NzbDrone.Core.Download.Clients.Putio
     {
         public Dictionary<string, PutioTorrentMetadata> Config { get; set; }
     }
+
+    public class PutioFileListingResponse : PutioGenericResponse
+    {
+        public static PutioFileListingResponse Empty()
+        {
+            return new PutioFileListingResponse
+            {
+                Files = new List<PutioFile>(),
+                Parent = new PutioFile
+                {
+                    Id = 0,
+                    Name = "Your Files"
+                }
+            };
+        }
+
+        public List<PutioFile> Files { get; set; }
+
+        public PutioFile Parent { get; set; }
+    }
 }