New: AniList Import List

This commit is contained in:
Xabis 2023-08-13 19:26:03 -04:00 committed by GitHub
parent efd19b6a6d
commit 465a584486
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 813 additions and 0 deletions

View File

@ -0,0 +1,155 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using NzbDrone.Common.Serializer;
namespace NzbDrone.Core.ImportLists.AniList
{
public enum AniListQuery
{
None,
QueryList
}
public static class AniListAPI
{
// The maximum items anilist will return is 50 per request.
// Therefore, pagination must be used to retrieve the full list.
private const string QueryList = @"
query ($id: String, $statusType: [MediaListStatus], $page: Int) {
Page(page: $page) {
pageInfo {
currentPage
lastPage
hasNextPage
}
mediaList(userName: $id, type: ANIME, status_in: $statusType) {
status
progress
media {
id
format
title {
userPreferred
romaji
}
status
episodes
startDate {
year
month
day
}
endDate {
year
month
day
}
}
}
}
}
";
public static string BuildQuery(AniListQuery query, object data)
{
var querySource = "";
if (query == AniListQuery.QueryList)
{
querySource = QueryList;
}
if (string.IsNullOrEmpty(querySource))
{
throw new Exception("Unknown Query Type");
}
var body = Json.ToJson(new
{
query = querySource,
variables = data
});
return body;
}
}
public class PageInfo
{
public int CurrentPage { get; set; }
public int LastPage { get; set; }
public bool HasNextPage { get; set; }
}
public class MediaTitles
{
public string UserPreferred { get; set; }
public string UserRomaji { get; set; }
}
public class MediaInfo
{
public int Id { get; set; }
public string Status { get; set; }
public string Format { get; set; }
public int? Episodes { get; set; }
public MediaTitles Title { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public YMDBlock StartDate { get; set; }
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public YMDBlock EndDate { get; set; }
}
public class MediaList
{
public MediaInfo Media { get; set; }
public string Status { get; set; }
public int Progress { get; set; }
}
public class MediaPage
{
public PageInfo PageInfo { get; set; }
public List<MediaList> MediaList { get; set; }
}
public class MediaPageData
{
public MediaPage Page { get; set; }
}
public class MediaPageResponse
{
public MediaPageData Data { get; set; }
}
public class YMDBlock
{
public int? Year { get; set; }
public int? Month { get; set; }
public int? Day { get; set; }
public DateTime? Convert()
{
if (Year == null)
{
return null;
}
return new DateTime((int)Year, Month ?? 1, Day ?? 1);
}
}
}

View File

@ -0,0 +1,111 @@
using System;
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.ImportLists.AniList
{
public abstract class AniListImportBase<TSettings> : HttpImportListBase<TSettings>
where TSettings : AniListSettingsBase<TSettings>, new()
{
public override ImportListType ListType => ImportListType.Other;
public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12);
public const string OAuthUrl = "https://anilist.co/api/v2/oauth/authorize";
public const string RedirectUri = "https://auth.servarr.com/v1/anilist_sonarr/auth";
public const string RenewUri = "https://auth.servarr.com/v1/anilist_sonarr/renew";
public const string ClientId = "13780";
protected IImportListRepository _importListRepository;
protected AniListImportBase(IImportListRepository netImportRepository,
IHttpClient httpClient,
IImportListStatusService importListStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(httpClient, importListStatusService, configService, parsingService, logger)
{
_importListRepository = netImportRepository;
}
public override object RequestAction(string action, IDictionary<string, string> query)
{
if (action == "startOAuth")
{
var request = new HttpRequestBuilder(OAuthUrl)
.AddQueryParam("client_id", ClientId)
.AddQueryParam("response_type", "code")
.AddQueryParam("redirect_uri", RedirectUri)
.AddQueryParam("state", query["callbackUrl"])
.Build();
return new
{
OauthUrl = request.Url.ToString()
};
}
else if (action == "getOAuthToken")
{
return new
{
accessToken = query["access_token"],
expires = DateTime.UtcNow.AddSeconds(int.Parse(query["expires_in"])),
refreshToken = query["refresh_token"],
};
}
return new { };
}
public override IList<ImportListItemInfo> Fetch()
{
CheckToken();
return base.Fetch();
}
protected void CheckToken()
{
Settings.Validate().Filter("AccessToken", "RefreshToken").ThrowOnError();
_logger.Trace("Access token expires at {0}", Settings.Expires);
if (Settings.Expires < DateTime.UtcNow.AddMinutes(5))
{
_logger.Trace("Refreshing Token");
Settings.Validate().Filter("RefreshToken").ThrowOnError();
var request = new HttpRequestBuilder(RenewUri)
.AddQueryParam("refresh_token", Settings.RefreshToken)
.Build();
try
{
var response = _httpClient.Get<RefreshRequestResponse>(request);
if (response != null && response.Resource != null)
{
var token = response.Resource;
Settings.AccessToken = token.AccessToken;
Settings.Expires = DateTime.UtcNow.AddSeconds(token.ExpiresIn);
Settings.RefreshToken = token.RefreshToken ?? Settings.RefreshToken;
if (Definition.Id > 0)
{
_importListRepository.UpdateSettings((ImportListDefinition)Definition);
}
}
}
catch (HttpException)
{
_logger.Warn($"Error refreshing access token");
}
}
}
}
}

View File

@ -0,0 +1,62 @@
using System;
using FluentValidation;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.ImportLists.AniList
{
public class AniListSettingsBaseValidator<TSettings> : AbstractValidator<TSettings>
where TSettings : AniListSettingsBase<TSettings>
{
public AniListSettingsBaseValidator()
{
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.AccessToken).NotEmpty()
.OverridePropertyName("SignIn")
.WithMessage("Must authenticate with AniList");
RuleFor(c => c.RefreshToken).NotEmpty()
.OverridePropertyName("SignIn")
.WithMessage("Must authenticate with AniList")
.When(c => c.AccessToken.IsNotNullOrWhiteSpace());
RuleFor(c => c.Expires).NotEmpty()
.OverridePropertyName("SignIn")
.WithMessage("Must authenticate with AniList")
.When(c => c.AccessToken.IsNotNullOrWhiteSpace() && c.RefreshToken.IsNotNullOrWhiteSpace());
}
}
public class AniListSettingsBase<TSettings> : IImportListSettings
where TSettings : AniListSettingsBase<TSettings>
{
protected virtual AbstractValidator<TSettings> Validator => new AniListSettingsBaseValidator<TSettings>();
public AniListSettingsBase()
{
BaseUrl = "https://graphql.anilist.co";
SignIn = "startOAuth";
}
public string BaseUrl { get; set; }
[FieldDefinition(0, Label = "Access Token", Type = FieldType.Textbox, Hidden = HiddenType.Hidden)]
public string AccessToken { get; set; }
[FieldDefinition(0, Label = "Refresh Token", Type = FieldType.Textbox, Hidden = HiddenType.Hidden)]
public string RefreshToken { get; set; }
[FieldDefinition(0, Label = "Expires", Type = FieldType.Textbox, Hidden = HiddenType.Hidden)]
public DateTime Expires { get; set; }
[FieldDefinition(99, Label = "Authenticate with AniList", Type = FieldType.OAuth)]
public string SignIn { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate((TSettings)this));
}
}
}

View File

@ -0,0 +1,42 @@
using Newtonsoft.Json;
namespace NzbDrone.Core.ImportLists.AniList
{
public static class MediaListStatus
{
public const string Current = "CURRENT";
public const string Planning = "PLANNING";
public const string Completed = "COMPLETED";
public const string Dropped = "DROPPED";
public const string Paused = "PAUSED";
public const string Repeating = "REPEATING";
}
public static class MediaStatus
{
public const string Finished = "FINISHED";
public const string Releasing = "RELEASING";
public const string Unreleased = "NOT_YET_RELEASED";
public const string Cancelled = "CANCELLED";
public const string Hiatus = "HIATUS";
}
public class RefreshRequestResponse
{
[JsonProperty("access_token")]
public string AccessToken { get; set; }
[JsonProperty("expires_in")]
public int ExpiresIn { get; set; }
[JsonProperty("refresh_token")]
public string RefreshToken { get; set; }
}
}

View File

@ -0,0 +1,153 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists.AniList.List
{
internal class AniListImport : AniListImportBase<AniListSettings>
{
public AniListImport(IImportListRepository netImportRepository,
IHttpClient httpClient,
IImportListStatusService importListStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(netImportRepository, httpClient, importListStatusService, configService, parsingService, logger)
{
}
public override string Name => "AniList List";
public override AniListRequestGenerator GetRequestGenerator()
{
return new AniListRequestGenerator()
{
Settings = Settings,
ClientId = ClientId
};
}
public override AniListParser GetParser()
{
return new AniListParser(Settings);
}
protected override IList<ImportListItemInfo> FetchItems(Func<IImportListRequestGenerator, ImportListPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
{
var releases = new List<ImportListItemInfo>();
var url = string.Empty;
try
{
var generator = GetRequestGenerator();
var parser = GetParser();
var pageIndex = 1;
var hasNextPage = false;
ImportListRequest currentRequest = null;
// Anilist caps the result list to 50 items at maximum per query, so the data must be pulled in batches.
// The number of pages are not known upfront, so the fetch logic must be changed to look at the returned page data.
do
{
// Build the query for the current page
currentRequest = generator.GetRequest(pageIndex);
url = currentRequest.Url.FullUri;
// Fetch and parse the response
var response = FetchImportListResponse(currentRequest);
var page = parser.ParseResponse(response, out var pageInfo).ToList();
releases.AddRange(page.Where(IsValidItem));
// Update page info
hasNextPage = pageInfo.HasNextPage; // server reports there is another page
pageIndex = pageInfo.CurrentPage + 1; // increment using the returned server index for the current page
}
while (hasNextPage);
_importListStatusService.RecordSuccess(Definition.Id);
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
webException.Status == WebExceptionStatus.ConnectFailure)
{
_importListStatusService.RecordConnectionFailure(Definition.Id);
}
else
{
_importListStatusService.RecordFailure(Definition.Id);
}
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
}
else
{
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (TooManyRequestsException ex)
{
if (ex.RetryAfter != TimeSpan.Zero)
{
_importListStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
}
else
{
_importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
}
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (HttpException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
}
catch (RequestLimitReachedException)
{
_importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (CloudFlareCaptchaException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
if (ex.IsExpired)
{
_logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in import list settings.", this);
}
else
{
_logger.Error(ex, "CAPTCHA token required for {0}, check import list settings.", this);
}
}
catch (ImportListException ex)
{
_importListStatusService.RecordFailure(Definition.Id);
_logger.Warn(ex, "{0}", url);
}
catch (Exception ex)
{
_importListStatusService.RecordFailure(Definition.Id);
ex.WithData("FeedUrl", url);
_logger.Error(ex, "An error occurred while processing feed. {0}", url);
}
return CleanupListItems(releases);
}
}
}

View File

@ -0,0 +1,109 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists.AniList.List
{
public class AniListParser : IParseImportListResponse
{
private readonly AniListSettings _settings;
public AniListParser(AniListSettings settings)
{
_settings = settings;
}
public virtual IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
{
return ParseResponse(importListResponse, out _);
}
public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse, out PageInfo pageInfo)
{
var result = new List<ImportListItemInfo>();
if (!PreProcess(importListResponse))
{
pageInfo = null;
return result;
}
var jsonResponse = STJson.Deserialize<MediaPageResponse>(importListResponse.Content);
if (jsonResponse?.Data?.Page?.MediaList == null)
{
pageInfo = null;
return result;
}
// Anilist currently does not support filtering this at the query level, they will get filtered out here.
var filtered = jsonResponse.Data.Page.MediaList
.Where(x => ValidateMediaStatus(x.Media));
foreach (var item in filtered)
{
var media = item.Media;
var entry = new ImportListItemInfo
{
AniListId = media.Id,
Title = media.Title.UserPreferred ?? media.Title.UserRomaji
};
result.Add(entry);
}
pageInfo = jsonResponse.Data.Page.PageInfo;
return result;
}
private bool ValidateMediaStatus(MediaInfo media)
{
if (media.Status == MediaStatus.Finished && _settings.ImportFinished)
{
return true;
}
if (media.Status == MediaStatus.Releasing && _settings.ImportReleasing)
{
return true;
}
if (media.Status == MediaStatus.Unreleased && _settings.ImportUnreleased)
{
return true;
}
if (media.Status == MediaStatus.Cancelled && _settings.ImportCancelled)
{
return true;
}
if (media.Status == MediaStatus.Hiatus && _settings.ImportHiatus)
{
return true;
}
return false;
}
protected virtual bool PreProcess(ImportListResponse netImportResponse)
{
if (netImportResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new ImportListException(netImportResponse, "Anilist API call resulted in an unexpected StatusCode [{0}]", netImportResponse.HttpResponse.StatusCode);
}
if (netImportResponse.HttpResponse.Headers.ContentType != null && !netImportResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
netImportResponse.HttpRequest.Headers.Accept != null && netImportResponse.HttpRequest.Headers.Accept.Contains("text/json"))
{
throw new ImportListException(netImportResponse, "Anilist API responded with html content. Site is likely blocked or unavailable.");
}
return true;
}
}
}

View File

@ -0,0 +1,89 @@
using System.Collections.Generic;
using System.Net.Http;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists.AniList.List
{
public class AniListRequestGenerator : IImportListRequestGenerator
{
public AniListSettings Settings { get; set; }
public string ClientId { get; set; }
public ImportListPageableRequestChain GetListItems()
{
return GetListItems(1);
}
public ImportListPageableRequestChain GetListItems(int page)
{
var pageableRequests = new ImportListPageableRequestChain();
pageableRequests.Add(GetRequestList(page));
return pageableRequests;
}
public ImportListRequest GetRequest(int page = 1)
{
var request = new ImportListRequest(Settings.BaseUrl.Trim(), HttpAccept.Json);
request.HttpRequest.Method = HttpMethod.Post;
request.HttpRequest.Headers.ContentType = "application/json";
request.HttpRequest.SetContent(AniListAPI.BuildQuery(AniListQuery.QueryList, new
{
id = Settings.Username,
statusType = BuildStatusFilter(),
page
}));
if (Settings.AccessToken.IsNotNullOrWhiteSpace())
{
request.HttpRequest.Headers.Add("Authorization", "Bearer " + Settings.AccessToken);
}
return request;
}
protected IEnumerable<ImportListRequest> GetRequestList(int page = 1)
{
yield return GetRequest(page);
}
private List<string> BuildStatusFilter()
{
var filters = new List<string>();
if (Settings.ImportCompleted)
{
filters.Add(MediaListStatus.Completed);
}
if (Settings.ImportCurrent)
{
filters.Add(MediaListStatus.Current);
}
if (Settings.ImportDropped)
{
filters.Add(MediaListStatus.Dropped);
}
if (Settings.ImportPaused)
{
filters.Add(MediaListStatus.Paused);
}
if (Settings.ImportPlanning)
{
filters.Add(MediaListStatus.Planning);
}
if (Settings.ImportRepeating)
{
filters.Add(MediaListStatus.Repeating);
}
return filters;
}
}
}

View File

@ -0,0 +1,70 @@
using FluentValidation;
using NzbDrone.Core.Annotations;
namespace NzbDrone.Core.ImportLists.AniList.List
{
public class AniListSettingsValidator : AniListSettingsBaseValidator<AniListSettings>
{
public AniListSettingsValidator()
: base()
{
RuleFor(c => c.Username).NotEmpty();
RuleFor(c => c.ImportCurrent).NotEmpty()
.WithMessage("At least one status type must be selected")
.When(c => !(c.ImportPlanning || c.ImportCompleted || c.ImportDropped || c.ImportPaused || c.ImportRepeating));
}
}
public class AniListSettings : AniListSettingsBase<AniListSettings>
{
public const string sectionImport = "Import List Status";
public AniListSettings()
: base()
{
ImportCurrent = true;
ImportPlanning = true;
ImportReleasing = true;
ImportFinished = true;
}
protected override AbstractValidator<AniListSettings> Validator => new AniListSettingsValidator();
[FieldDefinition(1, Label = "Username", HelpText = "Username for the List to import from")]
public string Username { get; set; }
[FieldDefinition(2, Label = "Import Watching", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: Currently Watching")]
public bool ImportCurrent { get; set; }
[FieldDefinition(3, Label = "Import Planning", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: Planning to Watch")]
public bool ImportPlanning { get; set; }
[FieldDefinition(4, Label = "Import Completed", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: Completed Watching")]
public bool ImportCompleted { get; set; }
[FieldDefinition(5, Label = "Import Dropped", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: Dropped")]
public bool ImportDropped { get; set; }
[FieldDefinition(6, Label = "Import Paused", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: On Hold")]
public bool ImportPaused { get; set; }
[FieldDefinition(7, Label = "Import Repeating", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "List: Currently Rewatching")]
public bool ImportRepeating { get; set; }
[FieldDefinition(8, Label = "Import Finished", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "Media: All episodes have aired")]
public bool ImportFinished { get; set; }
[FieldDefinition(9, Label = "Import Releasing", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "Media: Currently airing new episodes")]
public bool ImportReleasing { get; set; }
[FieldDefinition(10, Label = "Import Not Yet Released", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "Media: Airing has not yet started")]
public bool ImportUnreleased { get; set; }
[FieldDefinition(11, Label = "Import Cancelled", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "Media: Series is cancelled")]
public bool ImportCancelled { get; set; }
[FieldDefinition(12, Label = "Import Hiatus", Type = FieldType.Checkbox, Section = sectionImport, HelpText = "Media: Series on Hiatus")]
public bool ImportHiatus { get; set; }
}
}

View File

@ -93,6 +93,19 @@ namespace NzbDrone.Core.ImportLists
}
}
// Map by AniListId if we have it
if (report.TvdbId <= 0 && report.AniListId > 0)
{
var mappedSeries = _seriesSearchService.SearchForNewSeriesByAniListId(report.AniListId)
.FirstOrDefault();
if (mappedSeries != null)
{
report.TvdbId = mappedSeries.TvdbId;
report.Title = mappedSeries.Title;
}
}
// Map TVDb if we only have a series name
if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace())
{

View File

@ -7,5 +7,6 @@ namespace NzbDrone.Core.MetadataSource
{
List<Series> SearchForNewSeries(string title);
List<Series> SearchForNewSeriesByImdbId(string imdbId);
List<Series> SearchForNewSeriesByAniListId(int aniListId);
}
}

View File

@ -83,6 +83,13 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
return results;
}
public List<Series> SearchForNewSeriesByAniListId(int aniListId)
{
var results = SearchForNewSeries($"anilist:{aniListId}");
return results;
}
public List<Series> SearchForNewSeries(string title)
{
try

View File

@ -12,6 +12,7 @@ namespace NzbDrone.Core.Parser.Model
public int TmdbId { get; set; }
public string ImdbId { get; set; }
public int MalId { get; set; }
public int AniListId { get; set; }
public DateTime ReleaseDate { get; set; }
public override string ToString()