New: uTorrent differential api support to handle larger lists of torrents without hogging the api.

Fixes #1109

.gitignore

@@ -41,6 +41,9 @@ src/**/[Oo]bj/
 _ReSharper*
 _dotCover*
 
+# DevExpress CodeRush
+src/.cr/
+
 # NCrunch
 *.ncrunch*
 .*crunch*.local.xml

src/NzbDrone.Core.Test/Download/DownloadClientTests/UTorrentTests/UTorrentFixture.cs

@@ -84,7 +84,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
                         DownloadUrl = _downloadUrl,
                         RootDownloadPath = "somepath"
                     };
-            
+
             Mocker.GetMock<ITorrentFileInfoReader>()
                   .Setup(s => s.GetHashFromTorrentFile(It.IsAny<byte[]>()))
                   .Returns("CBC2F069FE8BB2F544EAE707D75BCD3DE9DCF951");
@@ -130,8 +130,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
                     PrepareClientToReturnQueuedItem();
                 });
         }
-        
+
-        protected virtual void GivenTorrents(List<UTorrentTorrent> torrents)
+        protected virtual void GivenTorrents(List<UTorrentTorrent> torrents, string cacheNumber = null)
         {
             if (torrents == null)
             {
@@ -139,13 +139,30 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
             }
 
             Mocker.GetMock<IUTorrentProxy>()
-                .Setup(s => s.GetTorrents(It.IsAny<UTorrentSettings>()))
+                .Setup(s => s.GetTorrents(It.IsAny<string>(), It.IsAny<UTorrentSettings>()))
-                .Returns(torrents);
+                .Returns(new UTorrentResponse { Torrents = torrents, CacheNumber = cacheNumber });
+        }
+
+        protected virtual void GivenDifferentialTorrents(string oldCacheNumber, List<UTorrentTorrent> changed, List<string> removed, string cacheNumber)
+        {
+            if (changed == null)
+            {
+                changed = new List<UTorrentTorrent>();
+            }
+
+            if (removed == null)
+            {
+                removed = new List<string>();
+            }
+
+            Mocker.GetMock<IUTorrentProxy>()
+                .Setup(s => s.GetTorrents(oldCacheNumber, It.IsAny<UTorrentSettings>()))
+                .Returns(new UTorrentResponse { TorrentsChanged = changed, TorrentsRemoved = removed, CacheNumber = cacheNumber });
         }
 
         protected void PrepareClientToReturnQueuedItem()
         {
-            GivenTorrents(new List<UTorrentTorrent> 
+            GivenTorrents(new List<UTorrentTorrent>
                 {
                     _queued
                 });
@@ -153,7 +170,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
 
         protected void PrepareClientToReturnDownloadingItem()
         {
-            GivenTorrents(new List<UTorrentTorrent> 
+            GivenTorrents(new List<UTorrentTorrent>
                 {
                     _downloading
                 });
@@ -161,7 +178,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
 
         protected void PrepareClientToReturnFailedItem()
         {
-            GivenTorrents(new List<UTorrentTorrent> 
+            GivenTorrents(new List<UTorrentTorrent>
                 {
                     _failed
                 });
@@ -296,13 +313,13 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
         public void should_return_status_with_outputdirs()
         {
             var configItems = new Dictionary<string, string>();
-            
+
             configItems.Add("dir_active_download_flag", "true");
             configItems.Add("dir_active_download", @"C:\Downloads\Downloading\utorrent".AsOsAgnostic());
             configItems.Add("dir_completed_download", @"C:\Downloads\Finished\utorrent".AsOsAgnostic());
             configItems.Add("dir_completed_download_flag", "true");
             configItems.Add("dir_add_label", "true");
-            
+
             Mocker.GetMock<IUTorrentProxy>()
                 .Setup(v => v.GetConfig(It.IsAny<UTorrentSettings>()))
                 .Returns(configItems);
@@ -353,5 +370,29 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.UTorrentTests
 
             id.Should().NotBeNullOrEmpty();
         }
+
+        [Test]
+        public void GetItems_should_query_with_cache_id_if_available()
+        {
+            _downloading.Status = UTorrentTorrentStatus.Started;
+
+            GivenTorrents(new List<UTorrentTorrent> { _downloading }, "abc");
+
+            var item1 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents(null, It.IsAny<UTorrentSettings>()), Times.Once());
+
+            GivenTorrents(new List<UTorrentTorrent> { _downloading, _queued }, "abc2");
+            GivenDifferentialTorrents("abc", new List<UTorrentTorrent> { _queued }, new List<string>(), "abc2");
+            GivenDifferentialTorrents("abc2", new List<UTorrentTorrent>(), new List<string>(), "abc2");
+
+            var item2 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents("abc", It.IsAny<UTorrentSettings>()), Times.Once());
+
+            var item3 = Subject.GetItems().Single();
+
+            Mocker.GetMock<IUTorrentProxy>().Verify(v => v.GetTorrents("abc2", It.IsAny<UTorrentSettings>()), Times.Once());
+        }
     }
 }

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs

@@ -12,14 +12,17 @@ using FluentValidation.Results;
 using System.Net;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
+using NzbDrone.Common.Cache;
 
 namespace NzbDrone.Core.Download.Clients.UTorrent
 {
     public class UTorrent : TorrentClientBase<UTorrentSettings>
     {
         private readonly IUTorrentProxy _proxy;
+        private readonly ICached<UTorrentTorrentCache> _torrentCache;
 
         public UTorrent(IUTorrentProxy proxy,
+                        ICacheManager cacheManager,
                         ITorrentFileInfoReader torrentFileInfoReader,
                         IHttpClient httpClient,
                         IConfigService configService,
@@ -29,6 +32,8 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
             : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger)
         {
             _proxy = proxy;
+
+            _torrentCache = cacheManager.GetCache<UTorrentTorrentCache>(GetType(), "differentialTorrents");
         }
 
         protected override string AddFromMagnetLink(RemoteEpisode remoteEpisode, string hash, string magnetLink)
@@ -72,12 +77,37 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
         }
 
         public override IEnumerable<DownloadClientItem> GetItems()
-        {            
+        {
             List<UTorrentTorrent> torrents;
 
             try
             {
-                torrents = _proxy.GetTorrents(Settings);
+                var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory);
+                var cache = _torrentCache.Find(cacheKey);
+
+                var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings);
+
+                if (cache != null && response.Torrents == null)
+                {
+                    var removedAndUpdated = new HashSet<string>(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved));
+
+                    torrents = cache.Torrents
+                        .Where(v => !removedAndUpdated.Contains(v.Hash))
+                        .Concat(response.TorrentsChanged)
+                        .ToList();
+                }
+                else
+                {
+                    torrents = response.Torrents;
+                }
+
+                cache = new UTorrentTorrentCache
+                {
+                    CacheID = response.CacheNumber,
+                    Torrents = torrents
+                };
+
+                _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15));
             }
             catch (DownloadClientException ex)
             {
@@ -122,7 +152,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
                     item.Status = DownloadItemStatus.Warning;
                     item.Message = "uTorrent is reporting an error";
                 }
-                else if (torrent.Status.HasFlag(UTorrentTorrentStatus.Loaded) && 
+                else if (torrent.Status.HasFlag(UTorrentTorrentStatus.Loaded) &&
                          torrent.Status.HasFlag(UTorrentTorrentStatus.Checked) && torrent.Remaining == 0 && torrent.Progress == 1.0)
                 {
                     item.Status = DownloadItemStatus.Completed;
@@ -239,7 +269,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
         {
             try
             {
-                _proxy.GetTorrents(Settings);
+                _proxy.GetTorrents(null, Settings);
             }
             catch (Exception ex)
             {

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs

@@ -15,7 +15,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
     {
         int GetVersion(UTorrentSettings settings);
         Dictionary<string, string> GetConfig(UTorrentSettings settings);
-        List<UTorrentTorrent> GetTorrents(UTorrentSettings settings);
+        UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings);
 
         void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings);
         void AddTorrentFromFile(string fileName, byte[] fileContent, UTorrentSettings settings);
@@ -70,14 +70,19 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
             return configuration;
         }
 
-        public List<UTorrentTorrent> GetTorrents(UTorrentSettings settings)
+        public UTorrentResponse GetTorrents(string cacheID, UTorrentSettings settings)
         {
             var requestBuilder = BuildRequest(settings)
                 .AddQueryParam("list", 1);
 
+            if (cacheID.IsNotNullOrWhiteSpace())
+            {
+                requestBuilder.AddQueryParam("cid", cacheID);
+            }
+
             var result = ProcessRequest(requestBuilder, settings);
 
-            return result.Torrents;
+            return result;
         }
 
         public void AddTorrentFromUrl(string torrentUrl, UTorrentSettings settings)

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentResponse.cs

@@ -12,6 +12,10 @@ namespace NzbDrone.Core.Download.Clients.UTorrent
         public List<object> RssFeeds { get; set; }
         public List<object> RssFilters { get; set; }
 
+        [JsonProperty(PropertyName = "torrentp")]
+        public List<UTorrentTorrent> TorrentsChanged { get; set; }
+        [JsonProperty(PropertyName = "torrentm")]
+        public List<string> TorrentsRemoved { get; set; }
         [JsonProperty(PropertyName = "torrentc")]
         public string CacheNumber { get; set; }
 

src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentTorrentCache.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace NzbDrone.Core.Download.Clients.UTorrent
+{
+    public class UTorrentTorrentCache
+    {
+        public string CacheID { get; set; }
+        
+        public List<UTorrentTorrent> Torrents { get; set; }
+    }
+}

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -451,6 +451,7 @@
     <Compile Include="Download\Clients\uTorrent\UTorrentResponse.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentSettings.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentTorrent.cs" />
+    <Compile Include="Download\Clients\uTorrent\UTorrentTorrentCache.cs" />
     <Compile Include="Download\Clients\uTorrent\UTorrentTorrentStatus.cs" />
     <Compile Include="Download\Clients\Vuze\Vuze.cs" />
     <Compile Include="Download\CompletedDownloadService.cs" />