Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement Remote File listing and local mapping

This commit is contained in:
Michael Feinbier 2023-09-29 17:14:47 +02:00
parent b68a9912f8
commit 12e5d1fad1
6 changed files with 236 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioFixture.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Http;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Clients.Putio;
@ -24,7 +25,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
[SetUp]
public void Setup()
{
_settings = new PutioSettings();
_settings = new PutioSettings
{
SaveParentId = "1"
};
Subject.Definition = new DownloadClientDefinition();
Subject.Definition.Settings = _settings;
@ -72,7 +76,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
Size = 1000,
Downloaded = 1000,
SaveParentId = 1,
FileId = 1
FileId = 2
};
_completed_different_parent = new PutioTorrent
@ -84,7 +88,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
Size = 1000,
Downloaded = 1000,
SaveParentId = 2,
FileId = 1
FileId = 3
};
_seeding = new PutioTorrent
@ -97,7 +101,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
Downloaded = 1000,
Uploaded = 1300,
SaveParentId = 1,
FileId = 2
FileId = 4
};
Mocker.GetMock<ITorrentFileInfoReader>()
@ -110,6 +114,10 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
Mocker.GetMock<IPutioProxy>()
.Setup(v => v.GetAccountSettings(It.IsAny<PutioSettings>()));
Mocker.GetMock<IPutioProxy>()
.Setup(v => v.GetFileListingResponse(It.IsAny<long>(), It.IsAny<PutioSettings>()))
.Returns(PutioFileListingResponse.Empty());
}
protected void GivenFailedDownload()
@ -121,18 +129,16 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
protected void GivenSuccessfulDownload()
{
Mocker.GetMock<IHttpClient>()
.Setup(s => s.Get(It.IsAny<HttpRequest>()))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), new byte[1000]));
/*
Mocker.GetMock<IPutioProxy>()
.Setup(s => s.AddTorrentFromUrl(It.IsAny<string>(), It.IsAny<PutioSettings>()))
.Callback(PrepareClientToReturnQueuedItem);
GivenRemoteFileStructure(new List<PutioFile>
{
new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
new PutioFile { Id = _seeding.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER },
}, new PutioFile { Id = 1, Name = "Downloads" });
Mocker.GetMock<IPutioProxy>()
.Setup(s => s.AddTorrentFromData(It.IsAny<byte[]>(), It.IsAny<PutioSettings>()))
.Callback(PrepareClientToReturnQueuedItem);
*/
// GivenRemoteFileStructure(new List<PutioFile>
// {
// new PutioFile { Id = _completed_different_parent.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
// }, new PutioFile { Id = 2, Name = "Downloads_new" });
}
protected virtual void GivenTorrents(List<PutioTorrent> torrents)
@ -144,6 +150,20 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
.Returns(torrents);
}
protected virtual void GivenRemoteFileStructure(List<PutioFile> files, PutioFile parentFile)
{
files ??= new List<PutioFile>();
var list = new PutioFileListingResponse { Files = files, Parent = parentFile };
Mocker.GetMock<IPutioProxy>()
.Setup(s => s.GetFileListingResponse(parentFile.Id, It.IsAny<PutioSettings>()))
.Returns(list);
Mocker.GetMock<IDiskProvider>()
.Setup(s => s.FolderExists(It.IsAny<string>()))
.Returns(true);
}
protected virtual void GivenMetadata(List<PutioTorrentMetadata> metadata)
{
metadata ??= new List<PutioTorrentMetadata>();
@ -170,12 +190,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
_seeding,
_completed_different_parent
});
GivenMetadata(new List<PutioTorrentMetadata>
{
PutioTorrentMetadata.fromTorrent(_completed, true),
PutioTorrentMetadata.fromTorrent(_seeding, true),
PutioTorrentMetadata.fromTorrent(_completed_different_parent, true),
});
GivenSuccessfulDownload();
var items = Subject.GetItems();
@ -184,9 +199,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
VerifyWarning(items.ElementAt(2));
VerifyCompleted(items.ElementAt(3));
VerifyCompleted(items.ElementAt(4));
VerifyCompleted(items.ElementAt(5));
items.Should().HaveCount(6);
items.Should().HaveCount(5);
}
[TestCase(1, 5)]
@ -203,6 +217,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
_seeding,
_completed_different_parent
});
GivenSuccessfulDownload();
_settings.SaveParentId = configuredParentId.ToString();
@ -225,7 +240,6 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
{
_queued
});
GivenMetadata(new List<PutioTorrentMetadata> { PutioTorrentMetadata.fromTorrent(_queued, true) });
var item = Subject.GetItems().Single();
@ -233,13 +247,41 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
}
[Test]
public void test_getItems_marks_non_existing_local_download_as_downloading()
public void test_getItems_path_for_folders()
{
GivenTorrents(new List<PutioTorrent> { _completed });
GivenMetadata(new List<PutioTorrentMetadata> { PutioTorrentMetadata.fromTorrent(_completed, false) });
GivenRemoteFileStructure(new List<PutioFile>
{
new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_FOLDER },
}, new PutioFile { Id = 1, Name = "Downloads" });
var item = Subject.GetItems().Single();
VerifyDownloading(item);
VerifyCompleted(item);
item.OutputPath.ToString().Should().ContainAll("Downloads", _title);
Mocker.GetMock<IPutioProxy>()
.Verify(s => s.GetFileListingResponse(1, It.IsAny<PutioSettings>()), Times.AtLeastOnce());
}
[Test]
public void test_getItems_path_for_files()
{
GivenTorrents(new List<PutioTorrent> { _completed });
GivenRemoteFileStructure(new List<PutioFile>
{
new PutioFile { Id = _completed.FileId, Name = _title, FileType = PutioFile.FILE_TYPE_VIDEO },
}, new PutioFile { Id = 1, Name = "Downloads" });
var item = Subject.GetItems().Single();
VerifyCompleted(item);
item.OutputPath.ToString().Should().Contain("Downloads");
item.OutputPath.ToString().Should().NotContain(_title);
Mocker.GetMock<IPutioProxy>()
.Verify(s => s.GetFileListingResponse(It.IsAny<long>(), It.IsAny<PutioSettings>()), Times.AtLeastOnce());
}
}
}

62
src/NzbDrone.Core.Test/Download/DownloadClientTests/PutioTests/PutioProxyFixtures.cs
View File

@ -59,6 +59,68 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.PutioTests
Assert.IsTrue(list["456"].Downloaded);
}
[Test]
public void test_GetFileListingResponse()
{
var json = @"{
""cursor"": null,
""files"": [
{
""file_type"": ""VIDEO"",
""id"": 111,
""name"": ""My.download.mkv"",
""parent_id"": 4711
},
{
""file_type"": ""FOLDER"",
""id"": 222,
""name"": ""Another-folder[dth]"",
""parent_id"": 4711
}
],
""parent"": {
""file_type"": ""FOLDER"",
""id"": 4711,
""name"": ""Incoming"",
""parent_id"": 0
},
""status"": ""OK"",
""total"": 2
}";
ClientGetWillReturn<PutioFileListingResponse>(json);
var response = Subject.GetFileListingResponse(4711, new PutioSettings());
Assert.That(response, Is.Not.Null);
Assert.AreEqual(response.Files.Count, 2);
Assert.AreEqual(4711, response.Parent.Id);
Assert.AreEqual(111, response.Files[0].Id);
Assert.AreEqual(222, response.Files[1].Id);
}
[Test]
public void test_GetFileListingResponse_empty()
{
var json = @"{
""cursor"": null,
""files"": [],
""parent"": {
""file_type"": ""FOLDER"",
""id"": 4711,
""name"": ""Incoming"",
""parent_id"": 0
},
""status"": ""OK"",
""total"": 0
}";
ClientGetWillReturn<PutioFileListingResponse>(json);
var response = Subject.GetFileListingResponse(4711, new PutioSettings());
Assert.That(response, Is.Not.Null);
Assert.AreEqual(response.Files.Count, 0);
}
private void ClientGetWillReturn<TResult>(string obj)
where TResult : new()
{

70
src/NzbDrone.Core/Download/Clients/Putio/Putio.cs
View File

@ -8,7 +8,6 @@ using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.MediaFiles.TorrentInfo;
using NzbDrone.Core.Organizer;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.RemotePathMappings;
using NzbDrone.Core.Validation;
@ -55,12 +54,20 @@ namespace NzbDrone.Core.Download.Clients.Putio
public override IEnumerable<DownloadClientItem> GetItems()
{
List<PutioTorrent> torrents;
Dictionary<string, PutioTorrentMetadata> metadata;
PutioFileListingResponse fileListingResponse;
try
{
torrents = _proxy.GetTorrents(Settings);
metadata = _proxy.GetAllTorrentMetadata(Settings);
if (Settings.SaveParentId.IsNotNullOrWhiteSpace())
{
fileListingResponse = _proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings);
}
else
{
fileListingResponse = _proxy.GetFileListingResponse(0, Settings);
}
}
catch (DownloadClientException ex)
{
@ -100,29 +107,31 @@ namespace NzbDrone.Core.Download.Clients.Putio
{
if (torrent.FileId != 0)
{
// How needs the output path need to look if we have remote files?
// Todo: make configurable? Behaviour might be different for users (rclone mount, vs sync/mv)
item.CanMoveFiles = false;
item.CanBeRemoved = false;
// check if we need to download the torrent from the remote
var title = FileNameBuilder.CleanFileName(torrent.Name);
var file = fileListingResponse.Files.FirstOrDefault(f => f.Id == torrent.FileId);
var parent = fileListingResponse.Parent;
// _diskProvider.FileExists(new OsPath())
/*
var file = _proxy.GetFile(torrent.FileId, Settings);
var torrentPath = "/completed/" + file.Name;
var outputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Url, new OsPath(torrentPath));
if (Settings.SaveParentId.IsNotNullOrWhiteSpace())
if (file == null || parent == null)
{
var directories = outputPath.FullPath.Split('\\', '/');
if (!directories.Contains(string.Format("{0}", Settings.SaveParentId)))
{
continue;
item.Message = string.Format("Did not find file {0} in remote listing", torrent.FileId);
item.Status = DownloadItemStatus.Warning;
}
else
{
var expectedPath = new OsPath(Settings.DownloadPath) + new OsPath(parent.Name);
if (file.IsFolder())
{
expectedPath += new OsPath(file.Name);
}
item.OutputPath = outputPath; // + torrent.Name;
*/
if (_diskProvider.FolderExists(expectedPath.FullPath))
{
item.OutputPath = expectedPath;
}
}
}
}
catch (DownloadClientException ex)
@ -170,12 +179,9 @@ namespace NzbDrone.Core.Download.Clients.Putio
public override DownloadClientInfo GetStatus()
{
var destDir = new OsPath(Settings.DownloadPath);
return new DownloadClientInfo
{
IsLocalhost = false,
OutputRootFolders = new List<OsPath> { destDir }
IsLocalhost = false
};
}
@ -183,6 +189,7 @@ namespace NzbDrone.Core.Download.Clients.Putio
{
failures.AddIfNotNull(TestFolder(Settings.DownloadPath, "DownloadPath"));
failures.AddIfNotNull(TestConnection());
failures.AddIfNotNull(TestRemoteParentFolder());
if (failures.Any())
{
return;
@ -229,6 +236,21 @@ namespace NzbDrone.Core.Download.Clients.Putio
return null;
}
private ValidationFailure TestRemoteParentFolder()
{
try
{
_proxy.GetFileListingResponse(long.Parse(Settings.SaveParentId), Settings);
}
catch (Exception ex)
{
_logger.Error(ex, ex.Message);
return new NzbDroneValidationFailure("SaveParentId", "This is not a valid folder in your account");
}
return null;
}
public override void RemoveItem(DownloadClientItem item, bool deleteData)
{
throw new NotImplementedException();

20
src/NzbDrone.Core/Download/Clients/Putio/PutioFile.cs Normal file
View File

@ -0,0 +1,20 @@
using Newtonsoft.Json;
namespace NzbDrone.Core.Download.Clients.Putio
{
public class PutioFile
{
public static string FILE_TYPE_FOLDER = "FOLDER";
public static string FILE_TYPE_VIDEO = "VIDEO";
public long Id { get; set; }
public string Name { get; set; }
[JsonProperty(PropertyName = "parent_id")]
public long ParentId { get; set; }
[JsonProperty(PropertyName = "file_type")]
public string FileType { get; set; }
public bool IsFolder()
{
return FileType == FILE_TYPE_FOLDER;
}
}
}

27
src/NzbDrone.Core/Download/Clients/Putio/PutioProxy.cs
View File

@ -13,10 +13,10 @@ namespace NzbDrone.Core.Download.Clients.Putio
List<PutioTorrent> GetTorrents(PutioSettings settings);
void AddTorrentFromUrl(string torrentUrl, PutioSettings settings);
void AddTorrentFromData(byte[] torrentData, PutioSettings settings);
void RemoveTorrent(string hash, PutioSettings settings);
void GetAccountSettings(PutioSettings settings);
public PutioTorrentMetadata GetTorrentMetadata(PutioTorrent torrent, PutioSettings settings);
public Dictionary<string, PutioTorrentMetadata> GetAllTorrentMetadata(PutioSettings settings);
public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings);
}
public class PutioProxy : IPutioProxy
@ -57,13 +57,6 @@ namespace NzbDrone.Core.Download.Clients.Putio
// ProcessRequest<PutioGenericResponse>(Method.POST, "transfers/add", arguments, settings);
}
public void RemoveTorrent(string hashString, PutioSettings settings)
{
// var arguments = new Dictionary<string, object>();
// arguments.Add("transfer_ids", new string[] { hashString });
// ProcessRequest<PutioGenericResponse>(Method.POST, "torrents/cancel", arguments, settings);
}
public void GetAccountSettings(PutioSettings settings)
{
Execute<PutioGenericResponse>(BuildRequest(HttpMethod.Get, "account/settings", settings));
@ -85,6 +78,24 @@ namespace NzbDrone.Core.Download.Clients.Putio
};
}
public PutioFileListingResponse GetFileListingResponse(long parentId, PutioSettings settings)
{
var request = BuildRequest(HttpMethod.Get, "files/list", settings);
request.AddQueryParam("parent_id", parentId);
request.AddQueryParam("per_page", 1000);
try
{
var response = Execute<PutioFileListingResponse>(request);
return response.Resource;
}
catch (DownloadClientException ex)
{
_logger.Error(ex, "Failed to get file listing response");
throw;
}
}
public Dictionary<string, PutioTorrentMetadata> GetAllTorrentMetadata(PutioSettings settings)
{
var metadata = Execute<PutioAllConfigResponse>(BuildRequest(HttpMethod.Get, "config", settings));

20
src/NzbDrone.Core/Download/Clients/Putio/PutioResponse.cs
View File

@ -25,4 +25,24 @@ namespace NzbDrone.Core.Download.Clients.Putio
{
public Dictionary<string, PutioTorrentMetadata> Config { get; set; }
}
public class PutioFileListingResponse : PutioGenericResponse
{
public static PutioFileListingResponse Empty()
{
return new PutioFileListingResponse
{
Files = new List<PutioFile>(),
Parent = new PutioFile
{
Id = 0,
Name = "Your Files"
}
};
}
public List<PutioFile> Files { get; set; }
public PutioFile Parent { get; set; }
}
}