Archive-Team
/
sonarr-repo-only
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Good riddance SyndicationFeed

This commit is contained in:
Mark McDowall 2013-08-05 19:45:57 -07:00
parent 57bb37a8cd
commit 7d43afd7b9
10 changed files with 117 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
NzbDrone.Core/Indexers/BasicRssParser.cs
View File

@ -2,8 +2,8 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using NLog;
using NzbDrone.Core.Parser.Model;
@ -25,27 +25,27 @@ namespace NzbDrone.Core.Indexers
public IEnumerable<ReportInfo> Process(Stream source)
{
//TODO: replace this BS with plain Linq to XML
var reader = new SyndicationFeedXmlReader(source);
var feed = SyndicationFeed.Load(reader).Items;
var xdoc = XDocument.Load(source);
var items = xdoc.Descendants("item");
var result = new List<ReportInfo>();
foreach (var syndicationItem in feed)
foreach (var item in items)
{
try
{
var parsedEpisode = ParseFeed(syndicationItem);
if (parsedEpisode != null)
var reportInfo = ParseFeedItem(item);
if (reportInfo != null)
{
parsedEpisode.NzbUrl = GetNzbUrl(syndicationItem);
parsedEpisode.NzbInfoUrl = GetNzbInfoUrl(syndicationItem);
result.Add(parsedEpisode);
reportInfo.NzbUrl = GetNzbUrl(item);
reportInfo.NzbInfoUrl = GetNzbInfoUrl(item);
result.Add(reportInfo);
}
}
catch (Exception itemEx)
{
itemEx.Data.Add("Item", syndicationItem.Title);
itemEx.Data.Add("Item", item.Title());
_logger.ErrorException("An error occurred while processing feed item", itemEx);
}
}
@ -54,37 +54,37 @@ namespace NzbDrone.Core.Indexers
}
protected virtual string GetTitle(SyndicationItem syndicationItem)
protected virtual string GetTitle(XElement item)
{
return syndicationItem.Title.Text;
return item.Title();
}
protected virtual string GetNzbUrl(SyndicationItem item)
protected virtual string GetNzbUrl(XElement item)
{
return item.Links[0].Uri.ToString();
return item.Links()[0];
}
protected virtual string GetNzbInfoUrl(SyndicationItem item)
protected virtual string GetNzbInfoUrl(XElement item)
{
return String.Empty;
}
protected virtual ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult)
protected virtual ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{
return currentResult;
}
private ReportInfo ParseFeed(SyndicationItem item)
private ReportInfo ParseFeedItem(XElement item)
{
var title = GetTitle(item);
var reportInfo = new ReportInfo();
reportInfo.Title = title;
reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate.Date).Days;
reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate().Date).Days;
reportInfo.ReleaseGroup = ParseReleaseGroup(title);
_logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title.Text);
_logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title());
return PostProcessor(item, reportInfo);
}

27
NzbDrone.Core/Indexers/Newznab/NewznabParser.cs
View File

@ -1,11 +1,15 @@
using System;
using System.ServiceModel.Syndication;
using System.Drawing;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Newznab
{
public class NewznabParser : BasicRssParser
{
private static XNamespace NEWZNAB = "http://www.newznab.com/DTD/2010/feeds/attributes/";
private readonly Newznab _newznabIndexer;
public NewznabParser(Newznab newznabIndexer)
@ -13,31 +17,22 @@ namespace NzbDrone.Core.Indexers.Newznab
_newznabIndexer = newznabIndexer;
}
protected override string GetNzbInfoUrl(SyndicationItem item)
protected override string GetNzbInfoUrl(XElement item)
{
return item.Id;
return item.Comments().Replace("#comments", "");
}
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult)
protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{
if (currentResult != null)
{
if (item.Links.Count > 1)
{
currentResult.Size = item.Links[1].Length;
}
var attributes = item.Elements(NEWZNAB + "attr");
var sizeElement = attributes.Single(e => e.Attribute("name").Value == "size");
currentResult.Indexer = GetName(item);
currentResult.Size = Convert.ToInt64(sizeElement.Attribute("value").Value);
}
return currentResult;
}
private string GetName(SyndicationItem item)
{
var hostname = item.Links[0].Uri.DnsSafeHost.ToLower();
return String.Format("{0}_{1}", _newznabIndexer.Name, hostname);
}
}
}

25
NzbDrone.Core/Indexers/NzbClub/NzbClubParser.cs
View File

@ -1,6 +1,6 @@
using System;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using NLog;
using NzbDrone.Core.Parser.Model;
@ -18,11 +18,11 @@ namespace NzbDrone.Core.Indexers.NzbClub
}
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult)
protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{
if (currentResult != null)
{
var match = SizeRegex.Match(item.Summary.Text);
var match = SizeRegex.Match(item.Description());
if (match.Success && match.Groups["size"].Success)
{
@ -30,26 +30,33 @@ namespace NzbDrone.Core.Indexers.NzbClub
}
else
{
logger.Warn("Couldn't parse size from {0}", item.Summary.Text);
logger.Warn("Couldn't parse size from {0}", item.Description());
}
}
return currentResult;
}
protected override string GetTitle(SyndicationItem syndicationItem)
protected override string GetTitle(XElement item)
{
var title = ParseHeader(syndicationItem.Title.Text);
var title = ParseHeader(item.Title());
if (String.IsNullOrWhiteSpace(title))
return syndicationItem.Title.Text;
return item.Title();
return title;
}
protected override string GetNzbInfoUrl(SyndicationItem item)
protected override string GetNzbInfoUrl(XElement item)
{
return item.Links[1].Uri.ToString();
return item.Links()[0];
}
protected override string GetNzbUrl(XElement item)
{
var enclosure = item.Element("enclosure");
return enclosure.Attribute("url").Value;
}
}
}

8
NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs
View File

@ -10,6 +10,14 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs
get { return "omgwtfnzbs"; }
}
public override IParseFeed Parser
{
get
{
return new OmgwtfnzbsParser();
}
}
public override IEnumerable<string> RecentFeed
{
get

15
NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs
View File

@ -1,21 +1,16 @@
using System;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Omgwtfnzbs
{
public class OmgwtfnzbsParser : BasicRssParser
{
protected override string GetNzbUrl(SyndicationItem item)
{
return item.Links[0].Uri.ToString();
}
protected override string GetNzbInfoUrl(SyndicationItem item)
protected override string GetNzbInfoUrl(XElement item)
{
//Todo: Me thinks I need to parse details to get this...
var match = Regex.Match(item.Summary.Text, @"(?:\<b\>View NZB\:\<\/b\>\s\<a\shref\=\"")(?<URL>.+)(?:\""\starget)",
var match = Regex.Match(item.Description(), @"(?:\<b\>View NZB\:\<\/b\>\s\<a\shref\=\"")(?<URL>.+)(?:\""\starget)",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
if (match.Success)
@ -26,11 +21,11 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs
return String.Empty;
}
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult)
protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{
if (currentResult != null)
{
var sizeString = Regex.Match(item.Summary.Text, @"Size:\<\/b\>\s\d+\.\d{1,2}\s\w{2}\<br \/\>", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value;
var sizeString = Regex.Match(item.Description(), @"(?:Size:\<\/b\>\s\d+\.)\d{1,2}\s\w{2}(?:\<br \/\>)", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value;
currentResult.Size = GetReportSize(sizeString);
}

98
NzbDrone.Core/Indexers/SyndicationFeedXmlReader.cs
View File

@ -1,98 +0,0 @@
//http://stackoverflow.com/questions/210375/problems-reading-rss-with-c-and-net-3-5
//https://connect.microsoft.com/VisualStudio/feedback/details/325421/syndicationfeed-load-fails-to-parse-datetime-against-a-real-world-feeds-ie7-can-read
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.ServiceModel.Syndication;
using System.Threading;
using System.Xml;
using NLog;
namespace NzbDrone.Core.Indexers
{
public class SyndicationFeedXmlReader : XmlTextReader
{
private static readonly Logger logger = LogManager.GetCurrentClassLogger();
private static readonly string[] rss20DateTimeHints = { "pubDate" };
private static readonly string[] atom10DateTimeHints = { "updated", "published", "lastBuildDate" };
private bool _isRss2DateTime;
private bool _isAtomDateTime;
private static readonly MethodInfo rss20FeedFormatterMethodInfo = typeof(Rss20FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Static);
private static readonly MethodInfo atom10FeedFormatterMethodInfo = typeof(Atom10FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Instance);
public SyndicationFeedXmlReader(Stream stream) : base(stream) { }
public override bool IsStartElement(string localname, string ns)
{
_isRss2DateTime = rss20DateTimeHints.Contains(localname);
_isAtomDateTime = atom10DateTimeHints.Contains(localname);
CheckForError();
return base.IsStartElement(localname, ns);
}
public override string ReadString()
{
var dateVal = base.ReadString();
try
{
if (_isRss2DateTime)
{
rss20FeedFormatterMethodInfo.Invoke(null, new object[] { dateVal, this });
}
if (_isAtomDateTime)
{
atom10FeedFormatterMethodInfo.Invoke(new Atom10FeedFormatter(), new object[] { dateVal, this });
}
}
catch (TargetInvocationException e)
{
DateTime parsedDate;
if (!DateTime.TryParse(dateVal, new CultureInfo("en-US"), DateTimeStyles.None, out parsedDate))
{
parsedDate = DateTime.UtcNow;
logger.WarnException("Unable to parse Feed date " + dateVal, e);
}
var currentCulture = Thread.CurrentThread.CurrentCulture;
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
dateVal = parsedDate.ToString("ddd, dd MMM yyyy HH:mm:ss zzz");
dateVal = dateVal.Remove(dateVal.LastIndexOf(':'), 1);
Thread.CurrentThread.CurrentCulture = currentCulture;
}
return dateVal;
}
private void CheckForError()
{
if (MoveToContent() == XmlNodeType.Element)
{
if (Name != "error")
return;
var message = "Error: ";
if (HasAttributes)
{
while (MoveToNextAttribute())
{
message += String.Format(" [{0}:{1}]", Name, Value);
}
}
logger.Error("Error in RSS feed: {0}", message);
throw new Exception(message);
}
}
}
}

11
NzbDrone.Core/Indexers/Wombles/WomblesParser.cs
View File

@ -1,21 +1,16 @@
using System.ServiceModel.Syndication;
using System.Xml.Linq;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Wombles
{
public class WomblesParser : BasicRssParser
{
protected override string GetNzbUrl(SyndicationItem item)
{
return item.Links[0].Uri.ToString();
}
protected override string GetNzbInfoUrl(SyndicationItem item)
protected override string GetNzbInfoUrl(XElement item)
{
return null;
}
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult)
protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{
if (currentResult != null)
{

52
NzbDrone.Core/Indexers/XElementExtensions.cs Normal file
View File

@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.Data.Odbc;
using System.Linq;
using System.Text;
using System.Xml.Linq;
namespace NzbDrone.Core.Indexers
{
public static class XElementExtensions
{
public static string Title(this XElement item)
{
return TryGetValue(item, "title", "Unknown");
}
public static DateTime PublishDate(this XElement item)
{
return DateTime.Parse(TryGetValue(item, "pubDate"));
}
public static List<String> Links(this XElement item)
{
var result = new List<String>();
var elements = item.Elements("link");
foreach (var link in elements)
{
result.Add(link.Value);
}
return result;
}
public static string Description(this XElement item)
{
return TryGetValue(item, "description");
}
public static string Comments(this XElement item)
{
return TryGetValue(item, "comments");
}
private static string TryGetValue(XElement item, string elementName, string defaultValue = "")
{
var element = item.Element(elementName);
return element != null ? element.Value : defaultValue;
}
}
}

4
NzbDrone.Core/NzbDrone.Core.csproj
View File

@ -207,6 +207,7 @@
<Compile Include="Indexers\IndexerSettingUpdatedEvent.cs" />
<Compile Include="Indexers\IndexerWithSetting.cs" />
<Compile Include="Indexers\RssSyncCommand.cs" />
<Compile Include="Indexers\XElementExtensions.cs" />
<Compile Include="Instrumentation\Commands\TrimLogCommand.cs" />
<Compile Include="Instrumentation\SetLoggingLevel.cs" />
<Compile Include="Jobs\TaskManager.cs" />
@ -435,9 +436,6 @@
<Compile Include="Indexers\IndexerFetchService.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Indexers\SyndicationFeedXmlReader.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="MediaFiles\MediaFileService.cs">
<SubType>Code</SubType>
</Compile>