New: uTorrent differential api support to handle larger lists of torrents without hogging the api.

Fixes #1109

.gitignore

@@ -41,6 +41,9 @@ src/**/[Oo]bj/
 _ReSharper*
 _dotCover*
 
+# DevExpress CodeRush
+src/.cr/
+
 # NCrunch
 *.ncrunch*
 .*crunch*.local.xml

src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs

@@ -131,7 +131,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
                 });
         }
 
-        protected virtual void GivenTorrents(List<UTorrentTorrent> torrents)
+        protected virtual void GivenTorrents(List<UTorrentTorrent> torrents, string cacheNumber = null)
         {
             if (torrents == null)
             {
@@ -139,8 +139,25 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
             }
 
             Mocker.GetMock<IUTorrentProxy>()
-                .Setup(s => s.GetTorrents(It.IsAny<UTorrentSettings>()))
+                .Setup(s => s.GetTorrents(It.IsAny<string>(), It.IsAny<UTorrentSettings>()))
-                .Returns(torrents);
+                .Returns(new UTorrentResponse { Torrents = torrents, CacheNumber = cacheNumber });
+        }
+
+        protected virtual void GivenDifferentialTorrents(string oldCacheNumber, List<UTorrentTorrent> changed, List<string> removed, string cacheNumber)
+        {
+            if (changed == null)
+            {
+                changed = new List<UTorrentTorrent>();
+            }
+
+            if (removed == null)
+            {
+                removed = new List<string>();
+            }
+
+            Mocker.GetMock<IUTorrentProxy>()
+                .Setup(s => s.GetTorrents(oldCacheNumber, It.IsAny<UTorrentSettings>()))
+                .Returns(new UTorrentResponse { TorrentsChanged = changed, TorrentsRemoved = removed, CacheNumber = cacheNumber });
         }
 
         protected void PrepareClientToReturnQueuedItem()
@@ -353,5 +370,29 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
 
             id.Should().NotBeNullOrEmpty();
         }
+
+        [Test]
+        public void GetItems_should_query_with_cache_id_if_available()
+        {
+            _downloading.Status = UTorrentTorrentStatus.Started;
+
+            GivenTorrents(new List<UTorrentTorrent> { _downloading }, "abc");
+
+            var item1 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents(null, It.IsAny<UTorrentSettings>()), Times.Once());
+
+            GivenTorrents(new List<UTorrentTorrent> { _downloading, _queued }, "abc2");
+            GivenDifferentialTorrents("abc", new List<UTorrentTorrent> { _queued }, new List<string>(), "abc2");
+            GivenDifferentialTorrents("abc2", new List<UTorrentTorrent>(), new List<string>(), "abc2");
+
+            var item2 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents("abc", It.IsAny<UTorrentSettings>()), Times.Once());
+
+            var item3 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents("abc2", It.IsAny<UTorrentSettings>()), Times.Once());
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs

@@ -12,14 +12,17 @@ using FluentValidation.Results;
 using System.Net;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
+using NzbDrone.Common.Cache;
 
 namespace NzbDrone.Core.Download.Clients.UTorrent
 {
     public class UTorrent : TorrentClientBase<UTorrentSettings>
     {
         private readonly IUTorrentProxy _proxy;
+        private readonly ICached<UTorrentTorrentCache> _torrentCache;
 
         public UTorrent(IUTorrentProxy proxy,
+                        ICacheManager cacheManager,
                         ITorrentFileInfoReader torrentFileInfoReader,
                         IHttpClient httpClient,
                         IConfigService configService,
@@ -29,6 +32,8 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
             : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger)
         {
             _proxy = proxy;
+
+            _torrentCache = cacheManager.GetCache<UTorrentTorrentCache>(GetType(), "differentialTorrents");
         }
 
         protected override string AddFromMagnetLink(RemoteEpisode remoteEpisode, string hash, string magnetLink)
@@ -77,7 +82,32 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
 
             try
             {
-                torrents = _proxy.GetTorrents(Settings);
+                var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory);
+                var cache = _torrentCache.Find(cacheKey);
+
+                var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings);
+
+                if (cache != null && response.Torrents == null)
+                {
+                    var removedAndUpdated = new HashSet<string>(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved));
+
+                    torrents = cache.Torrents
+                        .Where(v => !removedAndUpdated.Contains(v.Hash))
+                        .Concat(response.TorrentsChanged)
+                        .ToList();
+                }
+                else
+                {
+                    torrents = response.Torrents;
+                }
+
+                cache = new UTorrentTorrentCache
+                {
+                    CacheID = response.CacheNumber,
+                    Torrents = torrents
+                };
+
+                _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15));
             }
             catch (DownloadClientException ex)
             {
@@ -239,7 +269,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
         {
             try
             {
-                _proxy.GetTorrents(Settings);
+                _proxy.GetTorrents(null, Settings);
             }
             catch (Exception ex)
             {

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs

@@ -15,7 +15,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
     {
         int GetVersion(UTorrentSettings settings);
         Dictionary<string, string> GetConfig(UTorrentSettings settings);
-        List<UTorrentTorrent> GetTorrents(UTorrentSettings settings);
+        UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings);
 
         void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings);
         void AddTorrentFromFile(string fileName, byte[] fileContent, UTorrentSettings settings);
@@ -70,14 +70,19 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
             return configuration;
         }
 
-        public List<UTorrentTorrent> GetTorrents(UTorrentSettings settings)
+        public UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings)
         {
             var requestBuilder = BuildRequest(settings)
                 .AddQueryParam("list", 1);
 
+            if (cacheID.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.AddQueryParam("cid", cacheID);
+            }
+
             var result = ProcessRequest(requestBuilder, settings);
 
-            return result.Torrents;
+            return result;
         }
 
         public void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings)

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs

@@ -12,6 +12,10 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
         public List<object> RssFeeds { get; set; }
         public List<object> RssFilters { get; set; }
 
+        [JsonProperty(PropertyName = "torrentp")]
+        public List<UTorrentTorrent> TorrentsChanged { get; set; }
+        [JsonProperty(PropertyName = "torrentm")]
+        public List<string> TorrentsRemoved { get; set; }
         [JsonProperty(PropertyName = "torrentc")]
         public string CacheNumber { get; set; }
 

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace NzbDrone.Core.Download.Clients.UTorrent
+{
+    public class UTorrentTorrentCache
+    {
+        public string CacheID { get; set; }
+        
+        public List<UTorrentTorrent> Torrents { get; set; }
+    }
+}

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -451,6 +451,7 @@
     <Compile Include="Download\Clients\uTorrent\UTorrentResponse.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentSettings.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentTorrent.cs" />
+    <Compile Include="Download\Clients\uTorrent\UTorrentTorrentCache.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentTorrentStatus.cs" />
     <Compile Include="Download\Clients\Vuze\Vuze.cs" />
     <Compile Include="Download\CompletedDownloadService.cs" />