Fixed: Prevent duplicate parsing of extra files
This commit is contained in:
parent
2e96c4e798
commit
a621f0d49b
|
@ -14,18 +14,18 @@ namespace NzbDrone.Core.Extras
|
||||||
public class ExistingExtraFileService : IHandle<SeriesScannedEvent>
|
public class ExistingExtraFileService : IHandle<SeriesScannedEvent>
|
||||||
{
|
{
|
||||||
private readonly IDiskProvider _diskProvider;
|
private readonly IDiskProvider _diskProvider;
|
||||||
|
private readonly IDiskScanService _diskScanService;
|
||||||
private readonly List<IImportExistingExtraFiles> _existingExtraFileImporters;
|
private readonly List<IImportExistingExtraFiles> _existingExtraFileImporters;
|
||||||
private readonly List<IManageExtraFiles> _extraFileManagers;
|
|
||||||
private readonly Logger _logger;
|
private readonly Logger _logger;
|
||||||
|
|
||||||
public ExistingExtraFileService(IDiskProvider diskProvider,
|
public ExistingExtraFileService(IDiskProvider diskProvider,
|
||||||
|
IDiskScanService diskScanService,
|
||||||
List<IImportExistingExtraFiles> existingExtraFileImporters,
|
List<IImportExistingExtraFiles> existingExtraFileImporters,
|
||||||
List<IManageExtraFiles> extraFileManagers,
|
|
||||||
Logger logger)
|
Logger logger)
|
||||||
{
|
{
|
||||||
_diskProvider = diskProvider;
|
_diskProvider = diskProvider;
|
||||||
|
_diskScanService = diskScanService;
|
||||||
_existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList();
|
_existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList();
|
||||||
_extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList();
|
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,9 +41,8 @@ namespace NzbDrone.Core.Extras
|
||||||
|
|
||||||
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
||||||
|
|
||||||
var filesOnDisk = _diskProvider.GetFiles(series.Path, SearchOption.AllDirectories);
|
var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path);
|
||||||
var possibleExtraFiles = filesOnDisk.Where(c => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(c).ToLower()) &&
|
var possibleExtraFiles = _diskScanService.FilterFiles(series, filesOnDisk);
|
||||||
!c.StartsWith(Path.Combine(series.Path, "EXTRAS"))).ToList();
|
|
||||||
|
|
||||||
var filteredFiles = possibleExtraFiles;
|
var filteredFiles = possibleExtraFiles;
|
||||||
var importedFiles = new List<string>();
|
var importedFiles = new List<string>();
|
||||||
|
@ -55,7 +54,7 @@ namespace NzbDrone.Core.Extras
|
||||||
importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath)));
|
importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath)));
|
||||||
}
|
}
|
||||||
|
|
||||||
_logger.Info("Found {0} extra files", extraFiles);
|
_logger.Info("Found {0} extra files", extraFiles.Count);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using NzbDrone.Core.Extras.Files;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Extras
|
||||||
|
{
|
||||||
|
public class ImportExistingExtraFileFilterResult<TExtraFile>
|
||||||
|
where TExtraFile : ExtraFile, new()
|
||||||
|
{
|
||||||
|
public ImportExistingExtraFileFilterResult(List<TExtraFile> previouslyImported, List<string> filesOnDisk)
|
||||||
|
{
|
||||||
|
PreviouslyImported = previouslyImported;
|
||||||
|
FilesOnDisk = filesOnDisk;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<TExtraFile> PreviouslyImported { get; set; }
|
||||||
|
public List<string> FilesOnDisk { get; set; }
|
||||||
|
}
|
||||||
|
}
|
|
@ -21,7 +21,7 @@ namespace NzbDrone.Core.Extras
|
||||||
public abstract int Order { get; }
|
public abstract int Order { get; }
|
||||||
public abstract IEnumerable<ExtraFile> ProcessFiles(Series series, List<string> filesOnDisk, List<string> importedFiles);
|
public abstract IEnumerable<ExtraFile> ProcessFiles(Series series, List<string> filesOnDisk, List<string> importedFiles);
|
||||||
|
|
||||||
public virtual List<string> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
|
public virtual ImportExistingExtraFileFilterResult<TExtraFile> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
|
||||||
{
|
{
|
||||||
var seriesFiles = _extraFileService.GetFilesBySeries(series.Id);
|
var seriesFiles = _extraFileService.GetFilesBySeries(series.Id);
|
||||||
|
|
||||||
|
@ -30,12 +30,16 @@ namespace NzbDrone.Core.Extras
|
||||||
return Filter(series, filesOnDisk, importedFiles, seriesFiles);
|
return Filter(series, filesOnDisk, importedFiles, seriesFiles);
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<string> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
private ImportExistingExtraFileFilterResult<TExtraFile> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
||||||
{
|
{
|
||||||
var filteredFiles = filesOnDisk;
|
var previouslyImported = seriesFiles.IntersectBy(s => Path.Combine(series.Path, s.RelativePath), filesOnDisk, f => f, PathEqualityComparer.Instance).ToList();
|
||||||
|
var filteredFiles = filesOnDisk.Except(previouslyImported.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance)
|
||||||
|
.Except(importedFiles, PathEqualityComparer.Instance)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
filteredFiles = filteredFiles.Except(seriesFiles.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance).ToList();
|
// Return files that are already imported so they aren't imported again by other importers.
|
||||||
return filteredFiles.Except(importedFiles, PathEqualityComparer.Instance).ToList();
|
// Filter out files that were previously imported and as well as ones imported by other importers.
|
||||||
|
return new ImportExistingExtraFileFilterResult<TExtraFile>(previouslyImported, filteredFiles);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void Clean(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
private void Clean(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
|
||||||
|
|
|
@ -5,6 +5,7 @@ using NLog;
|
||||||
using NzbDrone.Common.Extensions;
|
using NzbDrone.Common.Extensions;
|
||||||
using NzbDrone.Core.Extras.Files;
|
using NzbDrone.Core.Extras.Files;
|
||||||
using NzbDrone.Core.Extras.Metadata.Files;
|
using NzbDrone.Core.Extras.Metadata.Files;
|
||||||
|
using NzbDrone.Core.Extras.Subtitles;
|
||||||
using NzbDrone.Core.Parser;
|
using NzbDrone.Core.Parser;
|
||||||
using NzbDrone.Core.Tv;
|
using NzbDrone.Core.Tv;
|
||||||
|
|
||||||
|
@ -42,10 +43,17 @@ namespace NzbDrone.Core.Extras.Metadata
|
||||||
_logger.Debug("Looking for existing metadata in {0}", series.Path);
|
_logger.Debug("Looking for existing metadata in {0}", series.Path);
|
||||||
|
|
||||||
var metadataFiles = new List<MetadataFile>();
|
var metadataFiles = new List<MetadataFile>();
|
||||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||||
|
|
||||||
foreach (var possibleMetadataFile in filteredFiles)
|
foreach (var possibleMetadataFile in filterResult.FilesOnDisk)
|
||||||
{
|
{
|
||||||
|
// Don't process files that have known Subtitle file extensions (saves a bit of unecessary processing)
|
||||||
|
|
||||||
|
if (SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(possibleMetadataFile)))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var consumer in _consumers)
|
foreach (var consumer in _consumers)
|
||||||
{
|
{
|
||||||
var metadata = consumer.FindMetadataFile(series, possibleMetadataFile);
|
var metadata = consumer.FindMetadataFile(series, possibleMetadataFile);
|
||||||
|
@ -90,7 +98,10 @@ namespace NzbDrone.Core.Extras.Metadata
|
||||||
_logger.Info("Found {0} existing metadata files", metadataFiles.Count);
|
_logger.Info("Found {0} existing metadata files", metadataFiles.Count);
|
||||||
_metadataFileService.Upsert(metadataFiles);
|
_metadataFileService.Upsert(metadataFiles);
|
||||||
|
|
||||||
return metadataFiles;
|
// Return files that were just imported along with files that were
|
||||||
|
// previously imported so previously imported files aren't imported twice
|
||||||
|
|
||||||
|
return metadataFiles.Concat(filterResult.PreviouslyImported);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,9 +38,9 @@ namespace NzbDrone.Core.Extras.Others
|
||||||
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
_logger.Debug("Looking for existing extra files in {0}", series.Path);
|
||||||
|
|
||||||
var extraFiles = new List<OtherExtraFile>();
|
var extraFiles = new List<OtherExtraFile>();
|
||||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||||
|
|
||||||
foreach (var possibleExtraFile in filteredFiles)
|
foreach (var possibleExtraFile in filterResult.FilesOnDisk)
|
||||||
{
|
{
|
||||||
var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series);
|
var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series);
|
||||||
|
|
||||||
|
@ -77,7 +77,10 @@ namespace NzbDrone.Core.Extras.Others
|
||||||
_logger.Info("Found {0} existing other extra files", extraFiles.Count);
|
_logger.Info("Found {0} existing other extra files", extraFiles.Count);
|
||||||
_otherExtraFileService.Upsert(extraFiles);
|
_otherExtraFileService.Upsert(extraFiles);
|
||||||
|
|
||||||
return extraFiles;
|
// Return files that were just imported along with files that were
|
||||||
|
// previously imported so previously imported files aren't imported twice
|
||||||
|
|
||||||
|
return extraFiles.Concat(filterResult.PreviouslyImported);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,9 +38,9 @@ namespace NzbDrone.Core.Extras.Subtitles
|
||||||
_logger.Debug("Looking for existing subtitle files in {0}", series.Path);
|
_logger.Debug("Looking for existing subtitle files in {0}", series.Path);
|
||||||
|
|
||||||
var subtitleFiles = new List<SubtitleFile>();
|
var subtitleFiles = new List<SubtitleFile>();
|
||||||
var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
|
var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
|
||||||
|
|
||||||
foreach (var possibleSubtitleFile in filteredFiles)
|
foreach (var possibleSubtitleFile in filterResult.FilesOnDisk)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(possibleSubtitleFile);
|
var extension = Path.GetExtension(possibleSubtitleFile);
|
||||||
|
|
||||||
|
@ -83,7 +83,10 @@ namespace NzbDrone.Core.Extras.Subtitles
|
||||||
_logger.Info("Found {0} existing subtitle files", subtitleFiles.Count);
|
_logger.Info("Found {0} existing subtitle files", subtitleFiles.Count);
|
||||||
_subtitleFileService.Upsert(subtitleFiles);
|
_subtitleFileService.Upsert(subtitleFiles);
|
||||||
|
|
||||||
return subtitleFiles;
|
// Return files that were just imported along with files that were
|
||||||
|
// previously imported so previously imported files aren't imported twice
|
||||||
|
|
||||||
|
return subtitleFiles.Concat(filterResult.PreviouslyImported);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,8 @@ namespace NzbDrone.Core.MediaFiles
|
||||||
{
|
{
|
||||||
void Scan(Series series);
|
void Scan(Series series);
|
||||||
string[] GetVideoFiles(string path, bool allDirectories = true);
|
string[] GetVideoFiles(string path, bool allDirectories = true);
|
||||||
|
string[] GetNonVideoFiles(string path, bool allDirectories = true);
|
||||||
|
List<string> FilterFiles(Series series, IEnumerable<string> files);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class DiskScanService :
|
public class DiskScanService :
|
||||||
|
@ -59,7 +61,7 @@ namespace NzbDrone.Core.MediaFiles
|
||||||
}
|
}
|
||||||
|
|
||||||
private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||||
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||||
|
|
||||||
public void Scan(Series series)
|
public void Scan(Series series)
|
||||||
{
|
{
|
||||||
|
@ -133,10 +135,25 @@ namespace NzbDrone.Core.MediaFiles
|
||||||
return mediaFileList.ToArray();
|
return mediaFileList.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<string> FilterFiles(Series series, IEnumerable<string> videoFiles)
|
public string[] GetNonVideoFiles(string path, bool allDirectories = true)
|
||||||
{
|
{
|
||||||
return videoFiles.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
|
_logger.Debug("Scanning '{0}' for non-video files", path);
|
||||||
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)));
|
|
||||||
|
var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
|
||||||
|
var filesOnDisk = _diskProvider.GetFiles(path, searchOption);
|
||||||
|
|
||||||
|
var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file).ToLower()))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
_logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path);
|
||||||
|
return mediaFileList.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<string> FilterFiles(Series series, IEnumerable<string> files)
|
||||||
|
{
|
||||||
|
return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
|
||||||
|
.Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)))
|
||||||
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void SetPermissions(string path)
|
private void SetPermissions(string path)
|
||||||
|
|
|
@ -495,6 +495,7 @@
|
||||||
<Compile Include="Extras\Files\ExtraFileRepository.cs" />
|
<Compile Include="Extras\Files\ExtraFileRepository.cs" />
|
||||||
<Compile Include="Extras\ExtraService.cs" />
|
<Compile Include="Extras\ExtraService.cs" />
|
||||||
<Compile Include="Extras\IImportExistingExtraFiles.cs" />
|
<Compile Include="Extras\IImportExistingExtraFiles.cs" />
|
||||||
|
<Compile Include="Extras\ImportExistingExtraFileFilterResult.cs" />
|
||||||
<Compile Include="Extras\ImportExistingExtraFilesBase.cs" />
|
<Compile Include="Extras\ImportExistingExtraFilesBase.cs" />
|
||||||
<Compile Include="Extras\Metadata\Files\MetadataFile.cs" />
|
<Compile Include="Extras\Metadata\Files\MetadataFile.cs" />
|
||||||
<Compile Include="Extras\Metadata\Files\MetadataFileRepository.cs" />
|
<Compile Include="Extras\Metadata\Files\MetadataFileRepository.cs" />
|
||||||
|
|
Loading…
Reference in New Issue