From c04ae9f1d037e418a7d461c15221b8b47df1f893 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 24 May 2014 12:47:01 +0200 Subject: [PATCH 1/4] Fixed: Newznab parser will attempt to use the usenetdate for age determination instead of the feed publish date. --- .../Indexers/Newznab/NewznabParser.cs | 16 ++++++++++++++++ src/NzbDrone.Core/Indexers/XElementExtensions.cs | 11 ++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs index 85fe2d4e4..cc1919c80 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using NzbDrone.Core.Parser.Model; +using System.Globalization; namespace NzbDrone.Core.Indexers.Newznab { @@ -19,6 +20,21 @@ namespace NzbDrone.Core.Indexers.Newznab return item.Comments().Replace("#comments", ""); } + protected override DateTime GetPublishDate(XElement item) + { + var attributes = item.Elements("attr").ToList(); + var usenetdateElement = attributes.SingleOrDefault(e => e.Attribute("name").Value.Equals("usenetdate", StringComparison.CurrentCultureIgnoreCase)); + + if (usenetdateElement != null) + { + var dateString = usenetdateElement.Attribute("value").Value; + + return XElementExtensions.ParseDate(dateString); + } + + return base.GetPublishDate(item); + } + protected override long GetSize(XElement item) { var attributes = item.Elements("attr").ToList(); diff --git a/src/NzbDrone.Core/Indexers/XElementExtensions.cs b/src/NzbDrone.Core/Indexers/XElementExtensions.cs index fc3f29dbc..fb24e526f 100644 --- a/src/NzbDrone.Core/Indexers/XElementExtensions.cs +++ b/src/NzbDrone.Core/Indexers/XElementExtensions.cs @@ -35,10 +35,8 @@ namespace NzbDrone.Core.Indexers return res; } - public static DateTime PublishDate(this XElement item) + public static DateTime ParseDate(string dateString) { - string dateString = item.TryGetValue("pubDate"); - try { DateTime result; @@ -56,6 +54,13 @@ namespace NzbDrone.Core.Indexers } } + public static DateTime PublishDate(this XElement item) + { + string dateString = item.TryGetValue("pubDate"); + + return ParseDate(dateString); + } + public static List Links(this XElement item) { var elements = item.Elements("link"); From 50303ce470e153e069346112be57ab70d22b7bfa Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Fri, 30 May 2014 19:38:52 +0200 Subject: [PATCH 2/4] Fixed: Checks full path for _UNPACK_ prefix so that full season downloads are properly checked for unpacking. --- .../NotUnpackingSpecificationFixture.cs | 2 +- .../NotUnpackingSpecification.cs | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecificationFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecificationFixture.cs index 155420917..57c444521 100644 --- a/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecificationFixture.cs +++ b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecificationFixture.cs @@ -36,7 +36,7 @@ namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Specifications private void GivenInWorkingFolder() { - _localEpisode.Path = @"C:\Test\Unsorted TV\_UNPACK_30.rock\30.rock.s01e01.avi".AsOsAgnostic(); + _localEpisode.Path = @"C:\Test\Unsorted TV\_UNPACK_30.rock\someSubFolder\30.rock.s01e01.avi".AsOsAgnostic(); } private void GivenLastWriteTimeUtc(DateTime time) diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecification.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecification.cs index 3c52b1bb6..97aa38e93 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecification.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotUnpackingSpecification.cs @@ -34,19 +34,25 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Specifications foreach (var workingFolder in _configService.DownloadClientWorkingFolders.Split('|')) { - if (Directory.GetParent(localEpisode.Path).Name.StartsWith(workingFolder)) + DirectoryInfo parent = Directory.GetParent(localEpisode.Path); + while (parent != null) { - if (OsInfo.IsMono) + if (parent.Name.StartsWith(workingFolder)) { - _logger.Debug("{0} is still being unpacked", localEpisode.Path); - return false; + if (OsInfo.IsMono) + { + _logger.Debug("{0} is still being unpacked", localEpisode.Path); + return false; + } + + if (_diskProvider.FileGetLastWriteUtc(localEpisode.Path) > DateTime.UtcNow.AddMinutes(-5)) + { + _logger.Debug("{0} appears to be unpacking still", localEpisode.Path); + return false; + } } - if (_diskProvider.FileGetLastWriteUtc(localEpisode.Path) > DateTime.UtcNow.AddMinutes(-5)) - { - _logger.Debug("{0} appears to be unpacking still", localEpisode.Path); - return false; - } + parent = parent.Parent; } } From e28123eefd76f67eb9ad4afca214e11a3ab99993 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Fri, 30 May 2014 22:36:38 +0200 Subject: [PATCH 3/4] Fixed: Search results from trakt are now sorted based on similarity with the search query. Using a Levenshtein distance algorithm. --- .../LevenshteinDistanceFixture.cs | 50 +++++++++++++++++ .../NzbDrone.Common.Test.csproj | 1 + src/NzbDrone.Common/LevenstheinExtensions.cs | 55 +++++++++++++++++++ src/NzbDrone.Common/NzbDrone.Common.csproj | 1 + .../MetadataSource/TraktProxy.cs | 4 +- 5 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 src/NzbDrone.Common.Test/LevenshteinDistanceFixture.cs create mode 100644 src/NzbDrone.Common/LevenstheinExtensions.cs diff --git a/src/NzbDrone.Common.Test/LevenshteinDistanceFixture.cs b/src/NzbDrone.Common.Test/LevenshteinDistanceFixture.cs new file mode 100644 index 000000000..27fe63480 --- /dev/null +++ b/src/NzbDrone.Common.Test/LevenshteinDistanceFixture.cs @@ -0,0 +1,50 @@ +using System; +using System.Diagnostics; +using System.IO; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Test.Common; + +namespace NzbDrone.Common.Test +{ + [TestFixture] + public class LevenshteinDistanceFixture : TestBase + { + [TestCase("", "", 0)] + [TestCase("abc", "abc", 0)] + [TestCase("abc", "abcd", 1)] + [TestCase("abcd", "abc", 1)] + [TestCase("abc", "abd", 1)] + [TestCase("abc", "adc", 1)] + [TestCase("abcdefgh", "abcghdef", 4)] + [TestCase("a.b.c.", "abc", 3)] + [TestCase("Agents Of SHIELD", "Marvel's Agents Of S.H.I.E.L.D.", 15)] + [TestCase("Agents of cracked", "Agents of shield", 6)] + [TestCase("ABCxxx", "ABC1xx", 1)] + [TestCase("ABC1xx", "ABCxxx", 1)] + public void LevenshteinDistance(String text, String other, Int32 expected) + { + text.LevenshteinDistance(other).Should().Be(expected); + } + + [TestCase("", "", 0)] + [TestCase("abc", "abc", 0)] + [TestCase("abc", "abcd", 1)] + [TestCase("abcd", "abc", 3)] + [TestCase("abc", "abd", 3)] + [TestCase("abc", "adc", 3)] + [TestCase("abcdefgh", "abcghdef", 8)] + [TestCase("a.b.c.", "abc", 0)] + [TestCase("Agents of shield", "Marvel's Agents Of S.H.I.E.L.D.", 9)] + [TestCase("Agents of shield", "Agents of cracked", 14)] + [TestCase("Agents of shield", "the shield", 24)] + [TestCase("ABCxxx", "ABC1xx", 3)] + [TestCase("ABC1xx", "ABCxxx", 3)] + public void LevenshteinDistanceClean(String text, String other, Int32 expected) + { + text.ToLower().LevenshteinDistanceClean(other.ToLower()).Should().Be(expected); + } + } +} diff --git a/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj b/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj index ae5e48bf8..e7f6a681f 100644 --- a/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj +++ b/src/NzbDrone.Common.Test/NzbDrone.Common.Test.csproj @@ -67,6 +67,7 @@ + diff --git a/src/NzbDrone.Common/LevenstheinExtensions.cs b/src/NzbDrone.Common/LevenstheinExtensions.cs new file mode 100644 index 000000000..3bc54d5b2 --- /dev/null +++ b/src/NzbDrone.Common/LevenstheinExtensions.cs @@ -0,0 +1,55 @@ +using System; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using ICSharpCode.SharpZipLib.Zip; + +namespace NzbDrone.Common +{ + public static class LevenstheinExtensions + { + public static Int32 LevenshteinDistance(this String text, String other, Int32 costInsert = 1, Int32 costDelete = 1, Int32 costSubstitute = 1) + { + if (text == other) return 0; + if (text.Length == 0) return other.Length * costInsert; + if (other.Length == 0) return text.Length * costDelete; + + Int32[] matrix = new Int32[other.Length + 1]; + + for (var i = 1; i < matrix.Length; i++) + { + matrix[i] = i * costInsert; + } + + for (var i = 0; i < text.Length; i++) + { + Int32 topLeft = matrix[0]; + matrix[0] = matrix[0] + costDelete; + + for (var j = 0; j < other.Length; j++) + { + Int32 top = matrix[j]; + Int32 left = matrix[j + 1]; + + var sumIns = top + costInsert; + var sumDel = left + costDelete; + var sumSub = topLeft + (text[i] == other[j] ? 0 : costSubstitute); + + topLeft = matrix[j + 1]; + matrix[j + 1] = Math.Min(Math.Min(sumIns, sumDel), sumSub); + } + } + + return matrix[other.Length]; + } + + public static Int32 LevenshteinDistanceClean(this String expected, String other) + { + expected = expected.ToLower().Replace(".", ""); + other = other.ToLower().Replace(".", ""); + + return expected.LevenshteinDistance(other, 1, 3, 3); + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index c75eeecc3..bab538ec7 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -114,6 +114,7 @@ + diff --git a/src/NzbDrone.Core/MetadataSource/TraktProxy.cs b/src/NzbDrone.Core/MetadataSource/TraktProxy.cs index 664778b63..22e11bec1 100644 --- a/src/NzbDrone.Core/MetadataSource/TraktProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/TraktProxy.cs @@ -35,7 +35,9 @@ namespace NzbDrone.Core.MetadataSource var restRequest = new RestRequest(GetSearchTerm(title) + "/30/seasons"); var response = client.ExecuteAndValidate>(restRequest); - return response.Select(MapSeries).ToList(); + return response.Select(MapSeries) + .OrderBy(v => title.LevenshteinDistanceClean(v.Title)) + .ToList(); } catch (WebException ex) { From 6d2fac5cd05f47a267be71c9362e381d81f01b51 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 31 May 2014 17:53:04 +0200 Subject: [PATCH 4/4] New: Adding new series by tvdbid: or slug: is now possible. --- .../MetadataSource/TraktProxy.cs | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/NzbDrone.Core/MetadataSource/TraktProxy.cs b/src/NzbDrone.Core/MetadataSource/TraktProxy.cs index 22e11bec1..0236a4169 100644 --- a/src/NzbDrone.Core/MetadataSource/TraktProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/TraktProxy.cs @@ -20,7 +20,7 @@ namespace NzbDrone.Core.MetadataSource { private readonly Logger _logger; private static readonly Regex CollapseSpaceRegex = new Regex(@"\s+", RegexOptions.Compiled); - private static readonly Regex InvalidSearchCharRegex = new Regex(@"(?:\*|\(|\)|'|!|@)", RegexOptions.Compiled); + private static readonly Regex InvalidSearchCharRegex = new Regex(@"(?:\*|\(|\)|'|!|@|\+)", RegexOptions.Compiled); public TraktProxy(Logger logger) { @@ -31,13 +31,43 @@ namespace NzbDrone.Core.MetadataSource { try { - var client = BuildClient("search", "shows"); - var restRequest = new RestRequest(GetSearchTerm(title) + "/30/seasons"); - var response = client.ExecuteAndValidate>(restRequest); + if (title.StartsWith("tvdb:") || title.StartsWith("tvdbid:") || title.StartsWith("slug:")) + { + try + { + var slug = title.Split(':')[1]; - return response.Select(MapSeries) - .OrderBy(v => title.LevenshteinDistanceClean(v.Title)) - .ToList(); + if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace)) + { + return new List(); + } + + var client = BuildClient("show", "summary"); + var restRequest = new RestRequest(GetSearchTerm(slug) + "/extended"); + var response = client.ExecuteAndValidate(restRequest); + + return new List { MapSeries(response) }; + } + catch (RestException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + return new List(); + } + + throw; + } + } + else + { + var client = BuildClient("search", "shows"); + var restRequest = new RestRequest(GetSearchTerm(title) + "/30/seasons"); + var response = client.ExecuteAndValidate>(restRequest); + + return response.Select(MapSeries) + .OrderBy(v => title.LevenshteinDistanceClean(v.Title)) + .ToList(); + } } catch (WebException ex) { @@ -172,7 +202,6 @@ namespace NzbDrone.Core.MetadataSource phrase = CollapseSpaceRegex.Replace(phrase, " ").Trim().ToLower(); phrase = phrase.Trim('-'); phrase = HttpUtility.UrlEncode(phrase); - return phrase; }