Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'pr/n721_ta264' into develop

This commit is contained in:
Taloth Saldono 2015-10-10 20:37:53 +02:00
parent 408d86245e 4be0fe1b76
commit 828071c1a5
8 changed files with 222 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build.ps1
View File

@ -218,6 +218,9 @@ Function PackageTests()
Write-Host "Adding NzbDrone.Core.dll.config (for dllmap)"
Copy-Item "$sourceFolder\NzbDrone.Core\NzbDrone.Core.dll.config" -Destination $testPackageFolder -Force
Write-Host "Copying CurlSharp libraries"
Copy-Item $sourceFolder\ExternalModules\CurlSharp\libs\i386\* $testPackageFolder
Write-Host "##teamcity[progressFinish 'Creating Test Package']"
}

3
build.sh
View File

@ -224,6 +224,9 @@ PackageTests()
echo "Adding CurlSharp.dll.config (for dllmap)"
cp $sourceFolder/NzbDrone.Common/CurlSharp.dll.config $testPackageFolder
echo "Copying CurlSharp libraries"
cp $sourceFolder/ExternalModules/CurlSharp/libs/i386/* $testPackageFolder
echo "##teamcity[progressFinish 'Creating Test Package']"
}

1
integration_mono.sh
View File

@ -5,3 +5,4 @@ NUNIT="$TESTDIR/NUnit.Runners.2.6.1/tools/nunit-console-x86.exe"
mono --debug --runtime=v4.0 $NUNIT $EXCLUDE -xml:NzbDrone.Api.Result.xml $TESTDIR/NzbDrone.Api.Test.dll
mono --debug --runtime=v4.0 $NUNIT $EXCLUDE -xml:NzbDrone.Core.Result.xml $TESTDIR/NzbDrone.Core.Test.dll
mono --debug --runtime=v4.0 $NUNIT $EXCLUDE -xml:NzbDrone.Integration.Result.xml $TESTDIR/NzbDrone.Integration.Test.dll
mono --debug --runtime=v4.0 $NUNIT $EXCLUDE -xml:NzbDrone.Common.Result.xml $TESTDIR/NzbDrone.Common.Test.dll

57
src/NzbDrone.Common.Test/Http/HttpClientFixture.cs
View File

@ -14,16 +14,33 @@ using Moq;
namespace NzbDrone.Common.Test.Http
{
[TestFixture]
[TestFixture(true)]
[TestFixture(false)]
[IntegrationTest]
public class HttpClientFixture : TestBase<HttpClient>
{
private bool _forceCurl;
public HttpClientFixture(bool forceCurl)
{
_forceCurl = forceCurl;
}
[SetUp]
public void SetUp()
{
Mocker.SetConstant<ICacheManager>(Mocker.Resolve<CacheManager>());
Mocker.SetConstant<IRateLimitService>(Mocker.Resolve<RateLimitService>());
Mocker.SetConstant<IEnumerable<IHttpRequestInterceptor>>(new IHttpRequestInterceptor[0]);
if (_forceCurl)
{
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<CurlHttpDispatcher>());
}
else
{
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<ManagedHttpDispatcher>());
}
}
[Test]
@ -36,6 +53,16 @@ namespace NzbDrone.Common.Test.Http
response.Content.Should().NotBeNullOrWhiteSpace();
}
[Test]
public void should_execute_https_get()
{
var request = new HttpRequest("https://eu.httpbin.org/get");
var response = Subject.Execute(request);
response.Content.Should().NotBeNullOrWhiteSpace();
}
[Test]
public void should_execute_typed_get()
{
@ -163,7 +190,7 @@ namespace NzbDrone.Common.Test.Http
var oldRequest = new HttpRequest("http://eu.httpbin.org/get");
oldRequest.AddCookie("my", "cookie");
var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<Logger>());
var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<ICacheManager>(), Mocker.Resolve<IRateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), Mocker.Resolve<Logger>());
oldClient.Should().NotBeSameAs(Subject);
@ -295,6 +322,32 @@ namespace NzbDrone.Common.Test.Http
Mocker.GetMock<IHttpRequestInterceptor>()
.Verify(v => v.PostResponse(It.IsAny<HttpResponse>()), Times.Once());
}
public void should_parse_malformed_cloudflare_cookie()
{
// the date is bad in the below - should be 13-Jul-2016
string malformedCookie = @"__cfduid=d29e686a9d65800021c66faca0a29b4261436890790; expires=Wed, 13-Jul-16 16:19:50 GMT; path=/; HttpOnly";
string url = "http://eu.httpbin.org/response-headers?Set-Cookie=" +
System.Uri.EscapeUriString(malformedCookie);
var requestSet = new HttpRequest(url);
requestSet.AllowAutoRedirect = false;
requestSet.StoreResponseCookie = true;
var responseSet = Subject.Get(requestSet);
var request = new HttpRequest("http://eu.httpbin.org/get");
var response = Subject.Get<HttpBinResource>(request);
response.Resource.Headers.Should().ContainKey("Cookie");
var cookie = response.Resource.Headers["Cookie"].ToString();
cookie.Should().Contain("__cfduid=d29e686a9d65800021c66faca0a29b4261436890790");
ExceptionVerification.IgnoreErrors();
}
}
public class HttpBinResource

5
src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -142,6 +142,9 @@
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" />
<PropertyGroup>
<PostBuildEvent Condition="'$(Configuration)|$(OS)' == 'Debug|Windows_NT'">xcopy /s /y "$(SolutionDir)\ExternalModules\CurlSharp\libs\i386\*" "$(TargetDir)"</PostBuildEvent>
</PropertyGroup>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

28
src/NzbDrone.Common/Http/CurlHttpClient.cs
View File

@ -6,8 +6,10 @@ using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using CurlSharp;
using NLog;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation;
@ -16,6 +18,7 @@ namespace NzbDrone.Common.Http
public class CurlHttpClient
{
private static Logger Logger = NzbDroneLogger.GetLogger(typeof(CurlHttpClient));
private static readonly Regex ExpiryDate = new Regex(@"(expires=)([^;]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public CurlHttpClient()
{
@ -64,7 +67,12 @@ namespace NzbDrone.Common.Http
curlEasy.HttpGet = webRequest.Method == "GET";
curlEasy.Post = webRequest.Method == "POST";
curlEasy.Put = webRequest.Method == "PUT";
curlEasy.Url = webRequest.RequestUri.ToString();
curlEasy.Url = webRequest.RequestUri.AbsoluteUri;
if (OsInfo.IsWindows)
{
curlEasy.CaInfo = "curl-ca-bundle.crt";
}
if (webRequest.CookieContainer != null)
{
@ -152,20 +160,34 @@ namespace NzbDrone.Common.Http
var webHeaderCollection = new WebHeaderCollection();
foreach (var header in headerString.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Skip(1))
// following a redirect we could have two sets of headers, so only process the last one
foreach (var header in headerString.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Reverse())
{
if (!header.Contains(":")) break;
webHeaderCollection.Add(header);
}
var setCookie = webHeaderCollection.Get("Set-Cookie");
if (setCookie != null && setCookie.Length > 0 && webRequest.CookieContainer != null)
{
webRequest.CookieContainer.SetCookies(webRequest.RequestUri, setCookie);
webRequest.CookieContainer.SetCookies(webRequest.RequestUri, FixSetCookieHeader(setCookie));
}
return webHeaderCollection;
}
private string FixSetCookieHeader(string setCookie)
{
// fix up the date if it was malformed
var setCookieClean = ExpiryDate.Replace(setCookie, delegate(Match match)
{
string format = "ddd, dd-MMM-yyyy HH:mm:ss";
DateTime dt = Convert.ToDateTime(match.Groups[2].Value);
return match.Groups[1].Value + dt.ToUniversalTime().ToString(format) + " GMT";
});
return setCookieClean;
}
private byte[] ProcessResponseStream(HttpWebRequest webRequest, Stream responseStream, WebHeaderCollection webHeaderCollection)
{
responseStream.Position = 0;

219
src/NzbDrone.Common/Http/HttpClient.cs
View File

@ -23,6 +23,119 @@ namespace NzbDrone.Common.Http
HttpResponse<T> Post<T>(HttpRequest request) where T : new();
}
public interface IHttpDispatcher
{
HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest);
}
public class ManagedHttpDispatcher : IHttpDispatcher
{
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
if (!request.Body.IsNullOrWhiteSpace())
{
var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray());
webRequest.ContentLength = bytes.Length;
using (var writeStream = webRequest.GetRequestStream())
{
writeStream.Write(bytes, 0, bytes.Length);
}
}
HttpWebResponse httpWebResponse;
try
{
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
httpWebResponse = (HttpWebResponse)e.Response;
if (httpWebResponse == null)
{
throw;
}
}
Byte[] data = null;
using (var responseStream = httpWebResponse.GetResponseStream())
{
if (responseStream != null)
{
data = responseStream.ToBytes();
}
}
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
}
}
public class CurlHttpDispatcher : IHttpDispatcher
{
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
var curlClient = new CurlHttpClient();
return curlClient.GetResponse(request, webRequest);
}
}
public class FallbackHttpDispatcher : IHttpDispatcher
{
private readonly Logger _logger;
private readonly ICached<bool> _curlTLSFallbackCache;
public FallbackHttpDispatcher(ICached<bool> curlTLSFallbackCache, Logger logger)
{
_logger = logger;
_curlTLSFallbackCache = curlTLSFallbackCache;
}
public HttpResponse GetResponse(HttpRequest request, HttpWebRequest webRequest)
{
ManagedHttpDispatcher managedDispatcher = new ManagedHttpDispatcher();
CurlHttpDispatcher curlDispatcher = new CurlHttpDispatcher();
if (OsInfo.IsMonoRuntime && webRequest.RequestUri.Scheme == "https")
{
if (!_curlTLSFallbackCache.Find(webRequest.RequestUri.Host))
{
try
{
return managedDispatcher.GetResponse(request, webRequest);
}
catch (Exception ex)
{
if (ex.ToString().Contains("The authentication or decryption has failed."))
{
_logger.Debug("https request failed in tls error for {0}, trying curl fallback.", webRequest.RequestUri.Host);
_curlTLSFallbackCache.Set(webRequest.RequestUri.Host, true);
}
else
{
throw;
}
}
}
if (CurlHttpClient.CheckAvailability())
{
return curlDispatcher.GetResponse(request, webRequest);
}
_logger.Trace("Curl not available, using default WebClient.");
}
return managedDispatcher.GetResponse(request, webRequest);
}
}
public class HttpClient : IHttpClient
{
private readonly Logger _logger;
@ -30,16 +143,23 @@ namespace NzbDrone.Common.Http
private readonly ICached<CookieContainer> _cookieContainerCache;
private readonly ICached<bool> _curlTLSFallbackCache;
private readonly List<IHttpRequestInterceptor> _requestInterceptors;
private readonly IHttpDispatcher _httpDispatcher;
public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger)
public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, Logger logger)
{
_logger = logger;
_rateLimitService = rateLimitService;
_requestInterceptors = requestInterceptors.ToList();
ServicePointManager.DefaultConnectionLimit = 12;
_httpDispatcher = httpDispatcher;
_cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
_curlTLSFallbackCache = cacheManager.GetCache<bool>(typeof(HttpClient), "curlTLSFallback");
}
public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger)
: this(requestInterceptors, cacheManager, rateLimitService, null, logger)
{
_httpDispatcher = new FallbackHttpDispatcher(cacheManager.GetCache<bool>(typeof(HttpClient), "curlTLSFallback"), _logger);
}
public HttpResponse Execute(HttpRequest request)
@ -79,7 +199,7 @@ namespace NzbDrone.Common.Http
PrepareRequestCookies(request, webRequest);
var response = ExecuteRequest(request, webRequest);
var response = _httpDispatcher.GetResponse(request, webRequest);
HandleResponseCookies(request, webRequest);
@ -89,8 +209,8 @@ namespace NzbDrone.Common.Http
if (!RuntimeInfoBase.IsProduction &&
(response.StatusCode == HttpStatusCode.Moved ||
response.StatusCode == HttpStatusCode.MovedPermanently ||
response.StatusCode == HttpStatusCode.Found))
response.StatusCode == HttpStatusCode.MovedPermanently ||
response.StatusCode == HttpStatusCode.Found))
{
_logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect.");
}
@ -129,7 +249,9 @@ namespace NzbDrone.Common.Http
{
persistentCookieContainer.Add(new Cookie(pair.Key, pair.Value, "/", request.Url.Host)
{
Expires = DateTime.UtcNow.AddHours(1)
// Use Now rather than UtcNow to work around Mono cookie expiry bug.
// See https://gist.github.com/ta264/7822b1424f72e5b4c961
Expires = DateTime.Now.AddHours(1)
});
}
}
@ -167,91 +289,6 @@ namespace NzbDrone.Common.Http
}
}
private HttpResponse ExecuteRequest(HttpRequest request, HttpWebRequest webRequest)
{
if (OsInfo.IsMonoRuntime && webRequest.RequestUri.Scheme == "https")
{
if (!_curlTLSFallbackCache.Find(webRequest.RequestUri.Host))
{
try
{
return ExecuteWebRequest(request, webRequest);
}
catch (Exception ex)
{
if (ex.ToString().Contains("The authentication or decryption has failed."))
{
_logger.Debug("https request failed in tls error for {0}, trying curl fallback.", webRequest.RequestUri.Host);
_curlTLSFallbackCache.Set(webRequest.RequestUri.Host, true);
}
else
{
throw;
}
}
}
if (CurlHttpClient.CheckAvailability())
{
return ExecuteCurlRequest(request, webRequest);
}
_logger.Trace("Curl not available, using default WebClient.");
}
return ExecuteWebRequest(request, webRequest);
}
private HttpResponse ExecuteCurlRequest(HttpRequest request, HttpWebRequest webRequest)
{
var curlClient = new CurlHttpClient();
return curlClient.GetResponse(request, webRequest);
}
private HttpResponse ExecuteWebRequest(HttpRequest request, HttpWebRequest webRequest)
{
if (!request.Body.IsNullOrWhiteSpace())
{
var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray());
webRequest.ContentLength = bytes.Length;
using (var writeStream = webRequest.GetRequestStream())
{
writeStream.Write(bytes, 0, bytes.Length);
}
}
HttpWebResponse httpWebResponse;
try
{
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
httpWebResponse = (HttpWebResponse)e.Response;
if (httpWebResponse == null)
{
throw;
}
}
byte[] data = null;
using (var responseStream = httpWebResponse.GetResponseStream())
{
if (responseStream != null)
{
data = responseStream.ToBytes();
}
}
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
}
public void DownloadFile(string url, string fileName)
{
try

4
src/NzbDrone.Common/Http/HttpHeader.cs
View File

@ -9,7 +9,7 @@ namespace NzbDrone.Common.Http
{
public class HttpHeader : Dictionary<string, object>
{
public HttpHeader(NameValueCollection headers)
public HttpHeader(NameValueCollection headers) : base(StringComparer.OrdinalIgnoreCase)
{
foreach (var key in headers.AllKeys)
{
@ -17,7 +17,7 @@ namespace NzbDrone.Common.Http
}
}
public HttpHeader()
public HttpHeader() : base(StringComparer.OrdinalIgnoreCase)
{
}