Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactored the HttpDispatchers.

This commit is contained in:
Taloth Saldono 2015-10-10 22:20:17 +02:00
parent e13c89521d
commit fe76d0f98f
7 changed files with 290 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/NzbDrone.Common.Test/Http/HttpClientFixture.cs
View File

@ -11,36 +11,22 @@ using NzbDrone.Test.Common.Categories;
using NLog;
using NzbDrone.Common.TPL;
using Moq;
using NzbDrone.Common.Http.Dispatchers;
namespace NzbDrone.Common.Test.Http
{
[TestFixture(true)]
[TestFixture(false)]
[IntegrationTest]
public class HttpClientFixture : TestBase<HttpClient>
[TestFixture(typeof(ManagedHttpDispatcher))]
[TestFixture(typeof(CurlHttpDispatcher))]
public class HttpClientFixture<TDispatcher> : TestBase<HttpClient> where TDispatcher : IHttpDispatcher
{
private bool _forceCurl;
public HttpClientFixture(bool forceCurl)
{
_forceCurl = forceCurl;
}
[SetUp]
public void SetUp()
{
Mocker.SetConstant<ICacheManager>(Mocker.Resolve<CacheManager>());
Mocker.SetConstant<IRateLimitService>(Mocker.Resolve<RateLimitService>());
Mocker.SetConstant<IEnumerable<IHttpRequestInterceptor>>(new IHttpRequestInterceptor[0]);
if (_forceCurl)
{
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<CurlHttpDispatcher>());
}
else
{
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<ManagedHttpDispatcher>());
}
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<TDispatcher>());
}
[Test]

124
src/NzbDrone.Common/Http/CurlHttpClient.cs → src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System;
using System.IO;
using System.IO.Compression;
using System.Linq;
@ -13,20 +12,13 @@ using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation;
namespace NzbDrone.Common.Http
namespace NzbDrone.Common.Http.Dispatchers
{
public class CurlHttpClient
public class CurlHttpDispatcher : IHttpDispatcher
{
private static Logger Logger = NzbDroneLogger.GetLogger(typeof(CurlHttpClient));
private static readonly Regex ExpiryDate = new Regex(@"(expires=)([^;]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public CurlHttpClient()
{
if (!CheckAvailability())
{
throw new ApplicationException("Curl failed to initialize.");
}
}
private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(CurlHttpDispatcher));
public static bool CheckAvailability()
{
@ -36,13 +28,23 @@ namespace NzbDrone.Common.Http
}
catch (Exception ex)
{
Logger.TraceException("Initializing curl failed", ex);
_logger.TraceException("Initializing curl failed", ex);
return false;
}
}
public HttpResponse GetResponse(HttpRequest httpRequest, HttpWebRequest webRequest)
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
{
if (!CheckAvailability())
{
throw new ApplicationException("Curl failed to initialize.");
}
if (request.NetworkCredential != null)
{
throw new NotImplementedException("Credentials not supported for curl dispatcher.");
}
lock (CurlGlobalHandle.Instance)
{
Stream responseStream = new MemoryStream();
@ -62,32 +64,46 @@ namespace NzbDrone.Common.Http
return s * n;
};
curlEasy.UserAgent = webRequest.UserAgent;
curlEasy.FollowLocation = webRequest.AllowAutoRedirect;
curlEasy.HttpGet = webRequest.Method == "GET";
curlEasy.Post = webRequest.Method == "POST";
curlEasy.Put = webRequest.Method == "PUT";
curlEasy.Url = webRequest.RequestUri.AbsoluteUri;
curlEasy.Url = request.Url.AbsoluteUri;
switch (request.Method)
{
case HttpMethod.GET:
curlEasy.HttpGet = true;
break;
case HttpMethod.POST:
curlEasy.Post = true;
break;
case HttpMethod.PUT:
curlEasy.Put = true;
break;
default:
throw new NotSupportedException(string.Format("HttpCurl method {0} not supported", request.Method));
}
curlEasy.UserAgent = UserAgentBuilder.UserAgent;
curlEasy.FollowLocation = request.AllowAutoRedirect;
if (OsInfo.IsWindows)
{
curlEasy.CaInfo = "curl-ca-bundle.crt";
}
if (webRequest.CookieContainer != null)
if (cookies != null)
{
curlEasy.Cookie = webRequest.CookieContainer.GetCookieHeader(webRequest.RequestUri);
curlEasy.Cookie = cookies.GetCookieHeader(request.Url);
}
if (!httpRequest.Body.IsNullOrWhiteSpace())
if (!request.Body.IsNullOrWhiteSpace())
{
// TODO: This might not go well with encoding.
curlEasy.PostFieldSize = httpRequest.Body.Length;
curlEasy.SetOpt(CurlOption.CopyPostFields, httpRequest.Body);
curlEasy.PostFieldSize = request.Body.Length;
curlEasy.SetOpt(CurlOption.CopyPostFields, request.Body);
}
// Yes, we have to keep a ref to the object to prevent corrupting the unmanaged state
using (var httpRequestHeaders = SerializeHeaders(webRequest))
using (var httpRequestHeaders = SerializeHeaders(request))
{
curlEasy.HttpHeader = httpRequestHeaders;
@ -99,60 +115,38 @@ namespace NzbDrone.Common.Http
}
}
var webHeaderCollection = ProcessHeaderStream(webRequest, headerStream);
var responseData = ProcessResponseStream(webRequest, responseStream, webHeaderCollection);
var webHeaderCollection = ProcessHeaderStream(request, cookies, headerStream);
var responseData = ProcessResponseStream(request, responseStream, webHeaderCollection);
var httpHeader = new HttpHeader(webHeaderCollection);
return new HttpResponse(httpRequest, httpHeader, responseData, (HttpStatusCode)curlEasy.ResponseCode);
return new HttpResponse(request, httpHeader, responseData, (HttpStatusCode)curlEasy.ResponseCode);
}
}
}
private CurlSlist SerializeHeaders(HttpWebRequest webRequest)
private CurlSlist SerializeHeaders(HttpRequest request)
{
if (webRequest.SendChunked)
if (!request.Headers.ContainsKey("Accept-Encoding"))
{
throw new NotSupportedException("Chunked transfer is not supported");
request.Headers.Add("Accept-Encoding", "gzip");
}
if (webRequest.ContentLength > 0)
if (request.Headers.ContentType == null)
{
webRequest.Headers.Add("Content-Length", webRequest.ContentLength.ToString());
request.Headers.ContentType = string.Empty;
}
if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip))
{
if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate))
{
webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
}
else
{
webRequest.Headers.Add("Accept-Encoding", "gzip");
}
}
else
{
if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate))
{
webRequest.Headers.Add("Accept-Encoding", "deflate");
}
}
var curlHeaders = new CurlSlist();
for (int i = 0; i < webRequest.Headers.Count; i++)
foreach (var header in request.Headers)
{
curlHeaders.Append(webRequest.Headers.GetKey(i) + ": " + webRequest.Headers.Get(i));
curlHeaders.Append(header.Key + ": " + header.Value.ToString());
}
curlHeaders.Append("Content-Type: " + webRequest.ContentType ?? string.Empty);
return curlHeaders;
}
private WebHeaderCollection ProcessHeaderStream(HttpWebRequest webRequest, Stream headerStream)
private WebHeaderCollection ProcessHeaderStream(HttpRequest request, CookieContainer cookies, Stream headerStream)
{
headerStream.Position = 0;
var headerData = headerStream.ToBytes();
@ -168,9 +162,9 @@ namespace NzbDrone.Common.Http
}
var setCookie = webHeaderCollection.Get("Set-Cookie");
if (setCookie != null && setCookie.Length > 0 && webRequest.CookieContainer != null)
if (setCookie != null && setCookie.Length > 0 && cookies != null)
{
webRequest.CookieContainer.SetCookies(webRequest.RequestUri, FixSetCookieHeader(setCookie));
cookies.SetCookies(request.Url, FixSetCookieHeader(setCookie));
}
return webHeaderCollection;
@ -188,22 +182,22 @@ namespace NzbDrone.Common.Http
return setCookieClean;
}
private byte[] ProcessResponseStream(HttpWebRequest webRequest, Stream responseStream, WebHeaderCollection webHeaderCollection)
private byte[] ProcessResponseStream(HttpRequest request, Stream responseStream, WebHeaderCollection webHeaderCollection)
{
responseStream.Position = 0;
if (responseStream.Length != 0 && webRequest.AutomaticDecompression != DecompressionMethods.None)
if (responseStream.Length != 0)
{
var encoding = webHeaderCollection["Content-Encoding"];
if (encoding != null)
{
if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip) && encoding.IndexOf("gzip") != -1)
if (encoding.IndexOf("gzip") != -1)
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
webHeaderCollection.Remove("Content-Encoding");
}
else if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate) && encoding.IndexOf("deflate") != -1)
else if (encoding.IndexOf("deflate") != -1)
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);

60
src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs Normal file
View File

@ -0,0 +1,60 @@
using System;
using System.Net;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
namespace NzbDrone.Common.Http.Dispatchers
{
public class FallbackHttpDispatcher : IHttpDispatcher
{
private readonly Logger _logger;
private readonly ICached<bool> _curlTLSFallbackCache;
private readonly ManagedHttpDispatcher _managedDispatcher;
private readonly CurlHttpDispatcher _curlDispatcher;
public FallbackHttpDispatcher(ICached<bool> curlTLSFallbackCache, Logger logger)
{
_logger = logger;
_curlTLSFallbackCache = curlTLSFallbackCache;
_managedDispatcher = new ManagedHttpDispatcher();
_curlDispatcher = new CurlHttpDispatcher();
}
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
{
if (OsInfo.IsMonoRuntime && request.Url.Scheme == "https")
{
if (!_curlTLSFallbackCache.Find(request.Url.Host))
{
try
{
return _managedDispatcher.GetResponse(request, cookies);
}
catch (Exception ex)
{
if (ex.ToString().Contains("The authentication or decryption has failed."))
{
_logger.Debug("https request failed in tls error for {0}, trying curl fallback.", request.Url.Host);
_curlTLSFallbackCache.Set(request.Url.Host, true);
}
else
{
throw;
}
}
}
if (CurlHttpDispatcher.CheckAvailability())
{
return _curlDispatcher.GetResponse(request, cookies);
}
_logger.Trace("Curl not available, using default WebClient.");
}
return _managedDispatcher.GetResponse(request, cookies);
}
}
}

19
src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.TPL;
namespace NzbDrone.Common.Http.Dispatchers
{
public interface IHttpDispatcher
{
HttpResponse GetResponse(HttpRequest request, CookieContainer cookies);
}
}

122
src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs Normal file
View File

@ -0,0 +1,122 @@
using System;
using System.Net;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.Http.Dispatchers
{
public class ManagedHttpDispatcher : IHttpDispatcher
{
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
{
var webRequest = (HttpWebRequest)WebRequest.Create(request.Url);
// Deflate is not a standard and could break depending on implementation.
// we should just stick with the more compatible Gzip
//http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
webRequest.Credentials = request.NetworkCredential;
webRequest.Method = request.Method.ToString();
webRequest.UserAgent = UserAgentBuilder.UserAgent;
webRequest.KeepAlive = false;
webRequest.AllowAutoRedirect = request.AllowAutoRedirect;
webRequest.ContentLength = 0;
webRequest.CookieContainer = cookies;
if (request.Headers != null)
{
AddRequestHeaders(webRequest, request.Headers);
}
if (!request.Body.IsNullOrWhiteSpace())
{
var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray());
webRequest.ContentLength = bytes.Length;
using (var writeStream = webRequest.GetRequestStream())
{
writeStream.Write(bytes, 0, bytes.Length);
}
}
HttpWebResponse httpWebResponse;
try
{
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
httpWebResponse = (HttpWebResponse)e.Response;
if (httpWebResponse == null)
{
throw;
}
}
byte[] data = null;
using (var responseStream = httpWebResponse.GetResponseStream())
{
if (responseStream != null)
{
data = responseStream.ToBytes();
}
}
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
}
protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)
{
foreach (var header in headers)
{
switch (header.Key)
{
case "Accept":
webRequest.Accept = header.Value.ToString();
break;
case "Connection":
webRequest.Connection = header.Value.ToString();
break;
case "Content-Length":
webRequest.ContentLength = Convert.ToInt64(header.Value);
break;
case "Content-Type":
webRequest.ContentType = header.Value.ToString();
break;
case "Date":
webRequest.Date = (DateTime)header.Value;
break;
case "Expect":
webRequest.Expect = header.Value.ToString();
break;
case "Host":
webRequest.Host = header.Value.ToString();
break;
case "If-Modified-Since":
webRequest.IfModifiedSince = (DateTime)header.Value;
break;
case "Range":
throw new NotImplementedException();
break;
case "Referer":
webRequest.Referer = header.Value.ToString();
break;
case "Transfer-Encoding":
webRequest.TransferEncoding = header.Value.ToString();
break;
case "User-Agent":
throw new NotSupportedException("User-Agent other than Sonarr not allowed.");
case "Proxy-Connection":
throw new NotImplementedException();
break;
default:
webRequest.Headers.Add(header.Key, header.Value.ToString());
break;
}
}
}
}
}

218
src/NzbDrone.Common/Http/HttpClient.cs
View File

@ -8,6 +8,7 @@ using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Dispatchers;
using NzbDrone.Common.TPL;
namespace NzbDrone.Common.Http
@ -23,119 +24,6 @@ namespace NzbDrone.Common.Http
HttpResponse<T> Post<T>(HttpRequest request) where T : new();
}
public interface IHttpDispatcher
{
HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest);
}
public class ManagedHttpDispatcher : IHttpDispatcher
{
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
if (!request.Body.IsNullOrWhiteSpace())
{
var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray());
webRequest.ContentLength = bytes.Length;
using (var writeStream = webRequest.GetRequestStream())
{
writeStream.Write(bytes, 0, bytes.Length);
}
}
HttpWebResponse httpWebResponse;
try
{
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
httpWebResponse = (HttpWebResponse)e.Response;
if (httpWebResponse == null)
{
throw;
}
}
Byte[] data = null;
using (var responseStream = httpWebResponse.GetResponseStream())
{
if (responseStream != null)
{
data = responseStream.ToBytes();
}
}
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
}
}
public class CurlHttpDispatcher : IHttpDispatcher
{
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
var curlClient = new CurlHttpClient();
return curlClient.GetResponse(request, webRequest);
}
}
public class FallbackHttpDispatcher : IHttpDispatcher
{
private readonly Logger _logger;
private readonly ICached<bool> _curlTLSFallbackCache;
public FallbackHttpDispatcher(ICached<bool> curlTLSFallbackCache, Logger logger)
{
_logger = logger;
_curlTLSFallbackCache = curlTLSFallbackCache;
}
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
ManagedHttpDispatcher managedDispatcher = new ManagedHttpDispatcher();
CurlHttpDispatcher curlDispatcher = new CurlHttpDispatcher();
if (OsInfo.IsMonoRuntime && webRequest.RequestUri.Scheme == "https")
{
if (!_curlTLSFallbackCache.Find(webRequest.RequestUri.Host))
{
try
{
return managedDispatcher.GetResponse(request, webRequest);
}
catch (Exception ex)
{
if (ex.ToString().Contains("The authentication or decryption has failed."))
{
_logger.Debug("https request failed in tls error for {0}, trying curl fallback.", webRequest.RequestUri.Host);
_curlTLSFallbackCache.Set(webRequest.RequestUri.Host, true);
}
else
{
throw;
}
}
}
if (CurlHttpClient.CheckAvailability())
{
return curlDispatcher.GetResponse(request, webRequest);
}
_logger.Trace("Curl not available, using default WebClient.");
}
return managedDispatcher.GetResponse(request, webRequest);
}
}
public class HttpClient : IHttpClient
{
private readonly Logger _logger;
@ -176,37 +64,23 @@ namespace NzbDrone.Common.Http
_logger.Trace(request);
var webRequest = (HttpWebRequest)WebRequest.Create(request.Url);
// Deflate is not a standard and could break depending on implementation.
// we should just stick with the more compatible Gzip
//http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
webRequest.Credentials = request.NetworkCredential;
webRequest.Method = request.Method.ToString();
webRequest.UserAgent = UserAgentBuilder.UserAgent;
webRequest.KeepAlive = false;
webRequest.AllowAutoRedirect = request.AllowAutoRedirect;
webRequest.ContentLength = 0;
var stopWatch = Stopwatch.StartNew();
if (request.Headers != null)
{
AddRequestHeaders(webRequest, request.Headers);
}
var cookies = PrepareRequestCookies(request);
PrepareRequestCookies(request, webRequest);
var response = _httpDispatcher.GetResponse(request, cookies);
var response = _httpDispatcher.GetResponse(request, webRequest);
HandleResponseCookies(request, webRequest);
HandleResponseCookies(request, cookies);
stopWatch.Stop();
_logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds);
foreach (var interceptor in _requestInterceptors)
{
response = interceptor.PostResponse(response);
}
if (!RuntimeInfoBase.IsProduction &&
(response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently ||
@ -229,15 +103,10 @@ namespace NzbDrone.Common.Http
}
}
foreach (var interceptor in _requestInterceptors)
{
response = interceptor.PostResponse(response);
}
return response;
}
private void PrepareRequestCookies(HttpRequest request, HttpWebRequest webRequest)
private CookieContainer PrepareRequestCookies(HttpRequest request)
{
lock (_cookieContainerCache)
{
@ -258,21 +127,15 @@ namespace NzbDrone.Common.Http
var requestCookies = persistentCookieContainer.GetCookies(request.Url);
if (requestCookies.Count == 0 && !request.StoreResponseCookie)
{
return;
}
var cookieContainer = new CookieContainer();
if (webRequest.CookieContainer == null)
{
webRequest.CookieContainer = new CookieContainer();
}
cookieContainer.Add(requestCookies);
webRequest.CookieContainer.Add(requestCookies);
return cookieContainer;
}
}
private void HandleResponseCookies(HttpRequest request, HttpWebRequest webRequest)
private void HandleResponseCookies(HttpRequest request, CookieContainer cookieContainer)
{
if (!request.StoreResponseCookie)
{
@ -283,7 +146,7 @@ namespace NzbDrone.Common.Http
{
var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
var cookies = webRequest.CookieContainer.GetCookies(request.Url);
var cookies = cookieContainer.GetCookies(request.Url);
persistentCookieContainer.Add(cookies);
}
@ -349,56 +212,5 @@ namespace NzbDrone.Common.Http
var response = Post(request);
return new HttpResponse<T>(response);
}
protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)
{
foreach (var header in headers)
{
switch (header.Key)
{
case "Accept":
webRequest.Accept = header.Value.ToString();
break;
case "Connection":
webRequest.Connection = header.Value.ToString();
break;
case "Content-Length":
webRequest.ContentLength = Convert.ToInt64(header.Value);
break;
case "Content-Type":
webRequest.ContentType = header.Value.ToString();
break;
case "Date":
webRequest.Date = (DateTime)header.Value;
break;
case "Expect":
webRequest.Expect = header.Value.ToString();
break;
case "Host":
webRequest.Host = header.Value.ToString();
break;
case "If-Modified-Since":
webRequest.IfModifiedSince = (DateTime)header.Value;
break;
case "Range":
throw new NotImplementedException();
break;
case "Referer":
webRequest.Referer = header.Value.ToString();
break;
case "Transfer-Encoding":
webRequest.TransferEncoding = header.Value.ToString();
break;
case "User-Agent":
throw new NotSupportedException("User-Agent other than Sonarr not allowed.");
case "Proxy-Connection":
throw new NotImplementedException();
break;
default:
webRequest.Headers.Add(header.Key, header.Value.ToString());
break;
}
}
}
}
}

5
src/NzbDrone.Common/NzbDrone.Common.csproj
View File

@ -141,7 +141,10 @@
<Compile Include="Extensions\UrlExtensions.cs" />
<Compile Include="Extensions\XmlExtentions.cs" />
<Compile Include="HashUtil.cs" />
<Compile Include="Http\CurlHttpClient.cs" />
<Compile Include="Http\Dispatchers\CurlHttpDispatcher.cs" />
<Compile Include="Http\Dispatchers\FallbackHttpDispatcher.cs" />
<Compile Include="Http\Dispatchers\IHttpDispatcher.cs" />
<Compile Include="Http\Dispatchers\ManagedHttpDispatcher.cs" />
<Compile Include="Http\GZipWebClient.cs">
<SubType>Component</SubType>
</Compile>