Added active detection for updatecheck so we know which os/runtime versions don't need to be supported anymore.

Fixed: Don't monitor newly added old episodes on tvdb if series was previously empty

src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs

@@ -2,9 +2,13 @@ using System;
 using System.IO;
 using System.IO.Compression;
 using System.Net;
+using System.Reflection;
+using NLog;
+using NLog.Fluent;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http.Proxy;
+using NzbDrone.Common.Instrumentation.Extensions;
 using NzbDrone.Common.Security;
 
 namespace NzbDrone.Common.Http.Dispatchers
@@ -14,12 +18,16 @@ namespace NzbDrone.Common.Http.Dispatchers
         private readonly IHttpProxySettingsProvider _proxySettingsProvider;
         private readonly ICreateManagedWebProxy _createManagedWebProxy;
         private readonly IUserAgentBuilder _userAgentBuilder;
+        private readonly IPlatformInfo _platformInfo;
+        private readonly Logger _logger;
 
-        public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICreateManagedWebProxy createManagedWebProxy, IUserAgentBuilder userAgentBuilder)
+        public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICreateManagedWebProxy createManagedWebProxy, IUserAgentBuilder userAgentBuilder, IPlatformInfo platformInfo, Logger logger)
         {
             _proxySettingsProvider = proxySettingsProvider;
             _createManagedWebProxy = createManagedWebProxy;
             _userAgentBuilder = userAgentBuilder;
+            _platformInfo = platformInfo;
+            _logger = logger;
         }
 
         public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
@@ -39,7 +47,7 @@ namespace NzbDrone.Common.Http.Dispatchers
                 //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
                 webRequest.AutomaticDecompression = DecompressionMethods.GZip;
             }
-            
+
             webRequest.Method = request.Method.ToString();
             webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent);
             webRequest.KeepAlive = request.ConnectionKeepAlive;
@@ -84,6 +92,9 @@ namespace NzbDrone.Common.Http.Dispatchers
 
                 if (httpWebResponse == null)
                 {
+                    // Workaround for mono not closing connections properly in certain situations.
+                    AbortWebRequest(webRequest);
+
                     // The default messages for WebException on mono are pretty horrible.
                     if (e.Status == WebExceptionStatus.NameResolutionFailure)
                     {
@@ -198,5 +209,36 @@ namespace NzbDrone.Common.Http.Dispatchers
                 }
             }
         }
+
+        // Workaround for mono not closing connections properly on timeouts
+        private void AbortWebRequest(HttpWebRequest webRequest)
+        {
+            // First affected version was mono 5.16
+            if (OsInfo.IsNotWindows && _platformInfo.Version >= new Version(5, 16))
+            {
+                try
+                {
+                    var currentOperationInfo = webRequest.GetType().GetField("currentOperation", BindingFlags.NonPublic | BindingFlags.Instance);
+                    var currentOperation = currentOperationInfo.GetValue(webRequest);
+
+                    if (currentOperation != null)
+                    {
+                        var responseStreamInfo = currentOperation.GetType().GetField("responseStream", BindingFlags.NonPublic | BindingFlags.Instance);
+                        var responseStream = responseStreamInfo.GetValue(currentOperation) as Stream;
+                        // Note that responseStream will likely be null once mono fixes it.
+                        responseStream?.Dispose();
+                    }
+                }
+                catch (Exception ex)
+                {
+                    // This can fail randomly on future mono versions that have been changed/fixed. Log to sentry and ignore.
+                    _logger.Trace()
+                           .Exception(ex)
+                           .Message("Unable to dispose responseStream on mono {0}", _platformInfo.Version)
+                           .WriteSentryWarn("MonoCloseWaitPatchFailed", ex.Message)
+                           .Write();
+                }
+            }
+        }
     }
 }

src/NzbDrone.Core.Test/Framework/CoreTest.cs

@@ -23,8 +23,8 @@ namespace NzbDrone.Core.Test.Framework
 
             Mocker.SetConstant<IHttpProxySettingsProvider>(new HttpProxySettingsProvider(Mocker.Resolve<ConfigService>()));
             Mocker.SetConstant<ICreateManagedWebProxy>(new ManagedWebProxyFactory(Mocker.Resolve<CacheManager>()));
-            Mocker.SetConstant<ManagedHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<UserAgentBuilder>()));
-            Mocker.SetConstant<CurlHttpDispatcher>(new CurlHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<NLog.Logger>()));
+            Mocker.SetConstant<ManagedHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<IPlatformInfo>(), TestLogger));
+            Mocker.SetConstant<CurlHttpDispatcher>(new CurlHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<UserAgentBuilder>(), TestLogger));
             Mocker.SetConstant<IHttpProvider>(new HttpProvider(TestLogger));
             Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<FallbackHttpDispatcher>(), Mocker.Resolve<UserAgentBuilder>(), TestLogger));
             Mocker.SetConstant<ISonarrCloudRequestBuilder>(new SonarrCloudRequestBuilder());

src/NzbDrone.Core.Test/TvTests/RefreshEpisodeServiceFixture.cs

@@ -9,6 +9,7 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Core.MetadataSource.SkyHook;
 using NzbDrone.Core.Tv;
 using NzbDrone.Core.Test.Framework;
+using NzbDrone.Test.Common;
 
 namespace NzbDrone.Core.Test.TvTests
 {
@@ -77,12 +78,14 @@ namespace NzbDrone.Core.Test.TvTests
         {
             Mocker.GetMock<IEpisodeService>().Setup(c => c.GetEpisodeBySeries(It.IsAny<int>()))
                 .Returns(new List<Episode>());
-
+            
             Subject.RefreshEpisodeInfo(GetSeries(), GetEpisodes());
 
             _insertedEpisodes.Should().HaveSameCount(GetEpisodes());
             _updatedEpisodes.Should().BeEmpty();
             _deletedEpisodes.Should().BeEmpty();
+
+            ExceptionVerification.ExpectedWarns(1);
         }
 
         [Test]
@@ -146,6 +149,63 @@ namespace NzbDrone.Core.Test.TvTests
             _updatedEpisodes.Should().OnlyContain(e => e.Monitored == true);
         }
 
+        [Test]
+        public void should_not_set_monitored_status_for_old_episodes_to_false_if_recent_enough()
+        {
+            var series = GetSeries();
+            series.Seasons = new List<Season>();
+            series.Seasons.Add(new Season { SeasonNumber = 1, Monitored = true });
+
+            var episodes = GetEpisodes().OrderBy(v => v.SeasonNumber).ThenBy(v => v.EpisodeNumber).Take(5).ToList();
+
+            episodes[1].AirDateUtc = DateTime.UtcNow.AddDays(-15);
+            episodes[2].AirDateUtc = DateTime.UtcNow.AddDays(-10);
+            episodes[3].AirDateUtc = DateTime.UtcNow.AddDays(1);
+
+            var existingEpisodes = episodes.Skip(4).ToList();
+
+            Mocker.GetMock<IEpisodeService>().Setup(c => c.GetEpisodeBySeries(It.IsAny<int>()))
+                .Returns(existingEpisodes);
+
+            Subject.RefreshEpisodeInfo(series, episodes);
+
+            _insertedEpisodes = _insertedEpisodes.OrderBy(v => v.EpisodeNumber).ToList();
+
+            _insertedEpisodes.Should().HaveCount(4);
+            _insertedEpisodes[0].Monitored.Should().Be(true);
+            _insertedEpisodes[1].Monitored.Should().Be(true);
+            _insertedEpisodes[2].Monitored.Should().Be(true);
+            _insertedEpisodes[3].Monitored.Should().Be(true);
+        }
+
+        [Test]
+        public void should_set_monitored_status_for_old_episodes_to_false_if_no_episodes_existed()
+        {
+            var series = GetSeries();
+            series.Seasons = new List<Season>();
+
+            var episodes = GetEpisodes().OrderBy(v => v.SeasonNumber).ThenBy(v => v.EpisodeNumber).Take(4).ToList();
+
+            episodes[1].AirDateUtc = DateTime.UtcNow.AddDays(-15);
+            episodes[2].AirDateUtc = DateTime.UtcNow.AddDays(-10);
+            episodes[3].AirDateUtc = DateTime.UtcNow.AddDays(1);
+
+            Mocker.GetMock<IEpisodeService>().Setup(c => c.GetEpisodeBySeries(It.IsAny<int>()))
+                .Returns(new List<Episode>());
+
+            Subject.RefreshEpisodeInfo(series, episodes);
+
+            _insertedEpisodes = _insertedEpisodes.OrderBy(v => v.EpisodeNumber).ToList();
+
+            _insertedEpisodes.Should().HaveSameCount(episodes);
+            _insertedEpisodes[0].Monitored.Should().Be(false);
+            _insertedEpisodes[1].Monitored.Should().Be(false);
+            _insertedEpisodes[2].Monitored.Should().Be(false);
+            _insertedEpisodes[3].Monitored.Should().Be(true);
+
+            ExceptionVerification.ExpectedWarns(1);
+        }
+
         [Test]
         public void should_remove_duplicate_remote_episodes_before_processing()
         {

src/NzbDrone.Core/Analytics/AnalyticsService.cs

@@ -1,22 +1,40 @@
-using NzbDrone.Common.EnvironmentInfo;
+using System;
+using System.Linq;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.History;
 
 namespace NzbDrone.Core.Analytics
 {
     public interface IAnalyticsService
     {
         bool IsEnabled { get; }
+        bool InstallIsActive { get; }
     }
 
     public class AnalyticsService : IAnalyticsService
     {
         private readonly IConfigFileProvider _configFileProvider;
+        private readonly IHistoryService _historyService;
 
-        public AnalyticsService(IConfigFileProvider configFileProvider)
+        public AnalyticsService(IHistoryService historyService, IConfigFileProvider configFileProvider)
         {
             _configFileProvider = configFileProvider;
+            _historyService = historyService;
         }
 
         public bool IsEnabled => _configFileProvider.AnalyticsEnabled && RuntimeInfo.IsProduction;
+
+        public bool InstallIsActive
+        {
+            get
+            {
+                var lastRecord = _historyService.Paged(new PagingSpec<History.History>() { Page = 0, PageSize = 1, SortKey = "date", SortDirection = SortDirection.Descending });
+                var monthAgo = DateTime.UtcNow.AddMonths(-1);
+
+                return lastRecord.Records.Any(v => v.Date > monthAgo); 
+            }
+        }
     }
-}
+}

src/NzbDrone.Core/Tv/EpisodeAddedService.cs

@@ -73,7 +73,9 @@ namespace NzbDrone.Core.Tv
                     return;
                 }
 
-                var previouslyAired = message.Added.Where(a => a.AirDateUtc.HasValue && a.AirDateUtc.Value.Before(DateTime.UtcNow.AddDays(1)) && a.Monitored).ToList();
+                var previouslyAired = message.Added.Where(a => a.AirDateUtc.HasValue
+                    && a.AirDateUtc.Value.Between(DateTime.UtcNow.AddDays(-14), DateTime.UtcNow.AddDays(1))
+                    && a.Monitored).ToList();
 
                 if (previouslyAired.Empty())
                 {

src/NzbDrone.Core/Tv/RefreshEpisodeService.cs

@@ -34,6 +34,7 @@ namespace NzbDrone.Core.Tv
 
             var existingEpisodes = _episodeService.GetEpisodeBySeries(series.Id);
             var seasons = series.Seasons;
+            var hasExisting = existingEpisodes.Any();
 
             var updateList = new List<Episode>();
             var newList = new List<Episode>();
@@ -82,6 +83,8 @@ namespace NzbDrone.Core.Tv
                 }
             }
 
+            UnmonitorReaddedEpisodes(series, newList, hasExisting);
+
             var allEpisodes = new List<Episode>();
             allEpisodes.AddRange(newList);
             allEpisodes.AddRange(updateList);
@@ -117,6 +120,41 @@ namespace NzbDrone.Core.Tv
             return season == null || season.Monitored;
         }
 
+
+        private void UnmonitorReaddedEpisodes(Series series, List<Episode> episodes, bool hasExisting)
+        {
+            if (series.AddOptions != null)
+            {
+                return;
+            }
+
+            var threshold = DateTime.UtcNow.AddDays(-14);
+
+            var oldEpisodes = episodes.Where(e => e.AirDateUtc.HasValue && e.AirDateUtc.Value.Before(threshold)).ToList();
+
+            if (oldEpisodes.Any())
+            {
+                if (hasExisting)
+                {
+                    _logger.Warn("Show {0} ({1}) had {2} old episodes appear, please check monitored status.", series.TvdbId, series.Title, oldEpisodes.Count);
+                }
+                else
+                {
+                    threshold = DateTime.UtcNow.AddDays(-1);
+
+                    foreach (var episode in episodes)
+                    {
+                        if (episode.AirDateUtc.HasValue && episode.AirDateUtc.Value.Before(threshold))
+                        {
+                            episode.Monitored = false;
+                        }
+                    }
+
+                    _logger.Warn("Show {0} ({1}) had {2} old episodes appear, unmonitored aired episodes to prevent unexpected downloads.", series.TvdbId, series.Title, oldEpisodes.Count);
+                }
+            }
+        }
+
         private void AdjustMultiEpisodeAirTime(Series series, IEnumerable<Episode> allEpisodes)
         {
             if (series.Network == "Netflix")

src/NzbDrone.Core/Update/UpdatePackageProvider.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using NzbDrone.Common.Cloud;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Http;
+using NzbDrone.Core.Analytics;
 
 namespace NzbDrone.Core.Update
 {
@@ -15,14 +16,16 @@ namespace NzbDrone.Core.Update
     public class UpdatePackageProvider : IUpdatePackageProvider
     {
         private readonly IHttpClient _httpClient;
-        private readonly IPlatformInfo _platformInfo;
         private readonly IHttpRequestBuilderFactory _requestBuilder;
+        private readonly IPlatformInfo _platformInfo;
+        private readonly IAnalyticsService _analyticsService;
 
-        public UpdatePackageProvider(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, IPlatformInfo platformInfo)
+        public UpdatePackageProvider(IHttpClient httpClient, ISonarrCloudRequestBuilder requestBuilder, IAnalyticsService analyticsService, IPlatformInfo platformInfo)
         {
-            _httpClient = httpClient;
             _platformInfo = platformInfo;
+            _analyticsService = analyticsService;
             _requestBuilder = requestBuilder.Services;
+            _httpClient = httpClient;
         }
 
         public UpdatePackage GetLatestUpdate(string branch, Version currentVersion)
@@ -32,10 +35,15 @@ namespace NzbDrone.Core.Update
                                          .AddQueryParam("version", currentVersion)
                                          .AddQueryParam("os", OsInfo.Os.ToString().ToLowerInvariant())
                                          .AddQueryParam("runtimeVer", _platformInfo.Version)
-                                         .SetSegment("branch", branch)
-                                         .Build();
+                                         .SetSegment("branch", branch);
 
-            var update = _httpClient.Get<UpdatePackageAvailable>(request).Resource;
+            if (_analyticsService.IsEnabled)
+            {
+                // Send if the system is active so we know which versions to deprecate/ignore
+                request.AddQueryParam("active", _analyticsService.InstallIsActive.ToString().ToLower());
+            }
+
+            var update = _httpClient.Get<UpdatePackageAvailable>(request.Build()).Resource;
 
             if (!update.Available) return null;
 
@@ -49,10 +57,15 @@ namespace NzbDrone.Core.Update
                                          .AddQueryParam("version", currentVersion)
                                          .AddQueryParam("os", OsInfo.Os.ToString().ToLowerInvariant())
                                          .AddQueryParam("runtimeVer", _platformInfo.Version)
-                                         .SetSegment("branch", branch)
-                                         .Build();
+                                         .SetSegment("branch", branch);
 
-            var updates = _httpClient.Get<List<UpdatePackage>>(request);
+            if (_analyticsService.IsEnabled)
+            {
+                // Send if the system is active so we know which versions to deprecate/ignore
+                request.AddQueryParam("active", _analyticsService.InstallIsActive.ToString().ToLower());
+            }
+
+            var updates = _httpClient.Get<List<UpdatePackage>>(request.Build());
 
             return updates.Resource;
         }