New: Fallback to libcurl/libssl on mono for https connections.

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "src/ExternalModules/CurlSharp"]
+	path = src/ExternalModules/CurlSharp
+	url = https://github.com/Sonarr/CurlSharp.git
+	branch = master

build.ps1

@@ -117,6 +117,9 @@ Function PackageMono()
     Write-Host "Adding NzbDrone.Core.dll.config (for dllmap)"
     Copy-Item "$sourceFolder\NzbDrone.Core\NzbDrone.Core.dll.config" $outputFolderMono
     
+    Write-Host "Adding CurlSharp.dll.config (for dllmap)"
+    Copy-Item "$sourceFolder\NzbDrone.Common\CurlSharp.dll.config" $outputFolderMono
+
     Write-Host Renaming NzbDrone.Console.exe to NzbDrone.exe
     Get-ChildItem $outputFolderMono -File -Filter "NzbDrone.exe*" -Recurse | foreach ($_) {remove-item $_.fullname}
 

src/ExternalModules/CurlSharp

@@ -0,0 +1 @@
+Subproject commit cfdbbbd9c6b9612c2756245049a8234ce87dc576

src/NzbDrone.Common.Test/Http/HttpClientFixture.cs

@@ -44,6 +44,17 @@ namespace NzbDrone.Common.Test.Http
             response.Resource.Url.Should().Be(request.Url.ToString());
         }
 
+        [Test]
+        public void should_execute_simple_post()
+        {
+            var request = new HttpRequest("http://eu.httpbin.org/post");
+            request.Body = "{ my: 1 }";
+
+            var response = Subject.Post<HttpBinResource>(request);
+
+            response.Resource.Data.Should().Be(request.Body);
+        }
+
         [TestCase("gzip")]
         public void should_execute_get_using_gzip(string compression)
         {
@@ -224,5 +235,6 @@ namespace NzbDrone.Common.Test.Http
         public Dictionary<string, object> Headers { get; set; }
         public string Origin { get; set; }
         public string Url { get; set; }
+        public string Data { get; set; }
     }
 }

src/NzbDrone.Common/CurlSharp.dll.config

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <!-- <dllmap os="osx" dll="libcurl.dll" target="libcurl.3.dylib"/> -->
+  <dllmap os="linux" dll="libcurl.dll" target="libcurl.so.3" />
+  <dllmap os="freebsd" dll="libcurl.dll" target="libcurl.so.3" />
+  <dllmap os="solaris" dll="libcurl.dll" target="libcurl.so.3" />
+</configuration>

src/NzbDrone.Common/Http/CurlHttpClient.cs

@@ -0,0 +1,233 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Runtime.InteropServices;
+using System.Text;
+using CurlSharp;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Instrumentation;
+
+namespace NzbDrone.Common.Http
+{
+    public class CurlHttpClient
+    {
+        private static Logger Logger = NzbDroneLogger.GetLogger(typeof(CurlHttpClient));
+
+        public CurlHttpClient()
+        {
+            if (!CheckAvailability())
+            {
+                throw new ApplicationException("Curl failed to initialize.");
+            }
+        }
+
+        public static bool CheckAvailability()
+        {
+            try
+            {
+                return CurlGlobalHandle.Instance.Initialize();
+            }
+            catch (Exception ex)
+            {
+                Logger.TraceException("Initializing curl failed", ex);
+                return false;
+            }
+        }
+
+        public HttpResponse GetResponse(HttpRequest httpRequest, HttpWebRequest webRequest)
+        {
+            Stream responseStream = new MemoryStream();
+            Stream headerStream = new MemoryStream();
+
+            var curlEasy = new CurlEasy();
+            curlEasy.AutoReferer = false;
+            curlEasy.WriteFunction = (b, s, n, o) =>
+            {
+                responseStream.Write(b, 0, s * n);
+                return s * n;
+            };
+            curlEasy.HeaderFunction = (b, s, n, o) =>
+            {
+                headerStream.Write(b, 0, s * n);
+                return s * n;
+            };
+
+            curlEasy.UserAgent = webRequest.UserAgent;
+            curlEasy.FollowLocation = webRequest.AllowAutoRedirect;
+            curlEasy.HttpGet = webRequest.Method == "GET";
+            curlEasy.Post = webRequest.Method == "POST";
+            curlEasy.Put = webRequest.Method == "PUT";
+            curlEasy.Url = webRequest.RequestUri.ToString();
+
+            if (webRequest.CookieContainer != null)
+            {
+                curlEasy.Cookie = webRequest.CookieContainer.GetCookieHeader(webRequest.RequestUri);
+            }
+
+            if (!httpRequest.Body.IsNullOrWhiteSpace())
+            {
+                // TODO: This might not go well with encoding.
+                curlEasy.PostFields = httpRequest.Body;
+                curlEasy.PostFieldSize = httpRequest.Body.Length;
+            }
+
+            curlEasy.HttpHeader = SerializeHeaders(webRequest);
+
+            var result = curlEasy.Perform();
+
+            if (result != CurlCode.Ok)
+            {
+                throw new WebException(string.Format("Curl Error {0} for Url {1}", result, curlEasy.Url));
+            }
+
+            var webHeaderCollection = ProcessHeaderStream(webRequest, headerStream);
+            var responseData = ProcessResponseStream(webRequest, responseStream, webHeaderCollection);
+
+            var httpHeader = new HttpHeader(webHeaderCollection);
+
+            return new HttpResponse(httpRequest, httpHeader, responseData, (HttpStatusCode)curlEasy.ResponseCode);
+        }
+
+        private CurlSlist SerializeHeaders(HttpWebRequest webRequest)
+        {
+            if (webRequest.SendChunked)
+            {
+                throw new NotSupportedException("Chunked transfer is not supported");
+            }
+
+            if (webRequest.ContentLength > 0)
+            {
+                webRequest.Headers.Add("Content-Length", webRequest.ContentLength.ToString());
+            }
+
+            if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip))
+            {
+                if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate))
+                {
+                    webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
+                }
+                else
+                {
+                    webRequest.Headers.Add("Accept-Encoding", "gzip");
+                }
+            }
+            else
+            {
+                if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate))
+                {
+                    webRequest.Headers.Add("Accept-Encoding", "deflate");
+                }
+            }
+
+
+            var curlHeaders = new CurlSlist();
+            for (int i = 0; i < webRequest.Headers.Count; i++)
+            {
+                curlHeaders.Append(webRequest.Headers.GetKey(i) + ": " + webRequest.Headers.Get(i));
+            }
+
+            curlHeaders.Append("Content-Type: " + webRequest.ContentType ?? string.Empty);
+
+            return curlHeaders;
+        }
+
+        private WebHeaderCollection ProcessHeaderStream(HttpWebRequest webRequest, Stream headerStream)
+        {
+            headerStream.Position = 0;
+            var headerData = headerStream.ToBytes();
+            var headerString = Encoding.ASCII.GetString(headerData);
+
+            var webHeaderCollection = new WebHeaderCollection();
+
+            foreach (var header in headerString.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Skip(1))
+            {
+                webHeaderCollection.Add(header);
+            }
+
+            var setCookie = webHeaderCollection.Get("Set-Cookie");
+            if (setCookie != null && setCookie.Length > 0 && webRequest.CookieContainer != null)
+            {
+                webRequest.CookieContainer.SetCookies(webRequest.RequestUri, setCookie);
+            }
+
+            return webHeaderCollection;
+        }
+
+        private byte[] ProcessResponseStream(HttpWebRequest webRequest, Stream responseStream, WebHeaderCollection webHeaderCollection)
+        {
+            responseStream.Position = 0;
+
+            if (responseStream.Length != 0 && webRequest.AutomaticDecompression != DecompressionMethods.None)
+            {
+                var encoding = webHeaderCollection["Content-Encoding"];
+                if (encoding != null)
+                {
+                    if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip) && encoding.IndexOf("gzip") != -1)
+                    {
+                        responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
+
+                        webHeaderCollection.Remove("Content-Encoding");
+                    }
+                    else if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate) && encoding.IndexOf("deflate") != -1)
+                    {
+                        responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
+
+                        webHeaderCollection.Remove("Content-Encoding");
+                    }
+                }
+            }
+
+            return responseStream.ToBytes();
+
+        }
+    }
+
+    internal sealed class CurlGlobalHandle : SafeHandle
+    {
+        public static readonly CurlGlobalHandle Instance = new CurlGlobalHandle();
+
+        private bool _initialized;
+        private bool _available;
+
+        protected override void Dispose(bool disposing)
+        {
+            base.Dispose(disposing);
+        }
+
+        private CurlGlobalHandle()
+            : base(IntPtr.Zero, true)
+        {
+
+        }
+
+        public bool Initialize()
+        {
+            if (_initialized)
+                return _available;
+
+            _initialized = true;
+            _available = Curl.GlobalInit(CurlInitFlag.All) == CurlCode.Ok;
+
+            return _available;
+        }
+
+        protected override bool ReleaseHandle()
+        {
+            if (_initialized && _available)
+            {
+                Curl.GlobalCleanup();
+                _available = false;
+            }
+            return true;
+        }
+
+        public override bool IsInvalid
+        {
+            get { return !_initialized || !_available; }
+        }
+    }
+}

src/NzbDrone.Common/Http/HttpClient.cs

@@ -26,6 +26,7 @@ namespace NzbDrone.Common.Http
         private readonly Logger _logger;
         private readonly IRateLimitService _rateLimitService;
         private readonly ICached<CookieContainer> _cookieContainerCache;
+        private readonly ICached<bool> _curlTLSFallbackCache;
 
         public HttpClient(ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger)
         {
@@ -34,6 +35,7 @@ namespace NzbDrone.Common.Http
             ServicePointManager.DefaultConnectionLimit = 12;
 
             _cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
+            _curlTLSFallbackCache = cacheManager.GetCache<bool>(typeof(HttpClient), "curlTLSFallback");
         }
 
         public HttpResponse Execute(HttpRequest request)
@@ -94,6 +96,74 @@ namespace NzbDrone.Common.Http
                 webRequest.CookieContainer.Add(cookieContainer.GetCookies(request.Url));
             }
 
+            var response = ExecuteRequest(request, webRequest);
+
+            stopWatch.Stop();
+
+            _logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds);
+
+            if (request.AllowAutoRedirect && !RuntimeInfoBase.IsProduction &&
+                (response.StatusCode == HttpStatusCode.Moved ||
+                response.StatusCode == HttpStatusCode.MovedPermanently ||
+                response.StatusCode == HttpStatusCode.Found))
+            {
+                _logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect.");
+            }
+
+            if (!request.SuppressHttpError && response.HasHttpError)
+            {
+                _logger.Warn("HTTP Error - {0}", response);
+                throw new HttpException(request, response);
+            }
+
+            return response;
+        }
+
+        private HttpResponse ExecuteRequest(HttpRequest request, HttpWebRequest webRequest)
+        {
+            if (OsInfo.IsMonoRuntime && webRequest.RequestUri.Scheme == "https")
+            {
+                if (!_curlTLSFallbackCache.Find(webRequest.RequestUri.Host))
+                {
+                    try
+                    {
+                        return ExecuteWebRequest(request, webRequest);
+                    }
+                    catch (Exception ex)
+                    {
+                        if (ex.ToString().Contains("The authentication or decryption has failed."))
+                        {
+                            _logger.Debug("https request failed in tls error for {0}, trying curl fallback.", webRequest.RequestUri.Host);
+
+                            _curlTLSFallbackCache.Set(webRequest.RequestUri.Host, true);
+                        }
+                        else
+                        {
+                            throw;
+                        }
+                    }
+                }
+
+                if (CurlHttpClient.CheckAvailability())
+                {
+                    return ExecuteCurlRequest(request, webRequest);
+                }
+
+                _logger.Trace("Curl not available, using default WebClient.");
+            }
+            
+            return ExecuteWebRequest(request, webRequest);
+        }
+
+        private HttpResponse ExecuteCurlRequest(HttpRequest request, HttpWebRequest webRequest)
+        {
+            var curlClient = new CurlHttpClient();
+
+            return curlClient.GetResponse(request, webRequest);
+        }
+
+        private HttpResponse ExecuteWebRequest(HttpRequest request, HttpWebRequest webRequest)
+        {
             if (!request.Body.IsNullOrWhiteSpace())
             {
                 var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray());
@@ -131,26 +201,7 @@ namespace NzbDrone.Common.Http
                 }
             }
 
-            stopWatch.Stop();
+            return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
-
-            var response = new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
-            _logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds);
-
-            if (request.AllowAutoRedirect && !RuntimeInfoBase.IsProduction &&
-                (response.StatusCode == HttpStatusCode.Moved ||
-                response.StatusCode == HttpStatusCode.MovedPermanently ||
-                response.StatusCode == HttpStatusCode.Found))
-            {
-                _logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect.");
-            }
-
-            if (!request.SuppressHttpError && response.HasHttpError)
-            {
-                _logger.Warn("HTTP Error - {0}", response);
-                throw new HttpException(request, response);
-            }
-
-            return response;
         }
 
         public void DownloadFile(string url, string fileName)

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -140,6 +140,7 @@
     <Compile Include="Extensions\StreamExtensions.cs" />
     <Compile Include="Extensions\XmlExtentions.cs" />
     <Compile Include="HashUtil.cs" />
+    <Compile Include="Http\CurlHttpClient.cs" />
     <Compile Include="Http\GZipWebClient.cs">
       <SubType>Component</SubType>
     </Compile>
@@ -196,6 +197,9 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="app.config" />
+    <None Include="CurlSharp.dll.config">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
@@ -208,6 +212,10 @@
     <Content Include="Expansive\license.txt" />
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\ExternalModules\CurlSharp\CurlSharp\CurlSharp.csproj">
+      <Project>{74420a79-cc16-442c-8b1e-7c1b913844f0}</Project>
+      <Name>CurlSharp</Name>
+    </ProjectReference>
     <ProjectReference Include="..\LogentriesNLog\LogentriesNLog.csproj">
       <Project>{9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}</Project>
       <Name>LogentriesNLog</Name>

src/NzbDrone.sln

@@ -88,6 +88,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LogentriesNLog", "Logentrie
 		{90D6E9FC-7B88-4E1B-B018-8FA742274558} = {90D6E9FC-7B88-4E1B-B018-8FA742274558}
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CurlSharp", "ExternalModules\CurlSharp\CurlSharp\CurlSharp.csproj", "{74420A79-CC16-442C-8B1E-7C1B913844F0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x86 = Debug|x86
@@ -268,6 +270,12 @@ Global
 		{9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Mono|x86.Build.0 = Release|x86
 		{9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Release|x86.ActiveCfg = Release|x86
 		{9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Release|x86.Build.0 = Release|x86
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Debug|x86.Build.0 = Debug|Any CPU
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Mono|x86.ActiveCfg = Release|Any CPU
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Mono|x86.Build.0 = Release|Any CPU
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Release|x86.ActiveCfg = Release|Any CPU
+		{74420A79-CC16-442C-8B1E-7C1B913844F0}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -300,6 +308,7 @@ Global
 		{411A9E0E-FDC6-4E25-828A-0C2CD1CD96F8} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA}
 		{90D6E9FC-7B88-4E1B-B018-8FA742274558} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA}
 		{9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA}
+		{74420A79-CC16-442C-8B1E-7C1B913844F0} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.0\lib\NET35;packages\Unity.2.1.505.2\lib\NET35