diff --git a/OpenDirectoryDownloader/Constants.cs b/OpenDirectoryDownloader/Constants.cs index 7e50c66e..4cbffe46 100644 --- a/OpenDirectoryDownloader/Constants.cs +++ b/OpenDirectoryDownloader/Constants.cs @@ -7,11 +7,20 @@ public class Constants public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss"; public const string Parameters_Password = "PASSWORD"; public const string Parameters_GdIndex_RootId = "GdIndex_RootId"; + public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode"; public class UserAgent { public const string Curl = "curl/7.55.1"; public const string Chrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3800.0 Safari/537.36"; } + + public class UriScheme + { + public const string Http = "http"; + public const string Https = "https"; + public const string Ftp = "ftp"; + public const string Ftps = "ftps"; + } } } diff --git a/OpenDirectoryDownloader/FtpParser.cs b/OpenDirectoryDownloader/FtpParser.cs index ba4ae538..d7768d44 100644 --- a/OpenDirectoryDownloader/FtpParser.cs +++ b/OpenDirectoryDownloader/FtpParser.cs @@ -16,45 +16,87 @@ namespace OpenDirectoryDownloader public class FtpParser { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); - private static readonly AsyncRetryPolicy RetryPolicy = Policy + private static readonly Random Jitterer = new Random(); + private static readonly AsyncRetryPolicy RetryPolicyNew = Policy .Handle() - .WaitAndRetryAsync(4, - sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + .WaitAndRetryAsync(100, + sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Min(16, Math.Pow(2, retryAttempt))) + TimeSpan.FromMilliseconds(Jitterer.Next(0, 200)), onRetry: (ex, span, retryCount, context) => { WebDirectory webDirectory = context["WebDirectory"] as WebDirectory; - Logger.Warn($"[{context["Processor"]}] Error {ex.Message} retrieving on try {retryCount} for url '{webDirectory.Url}'. Waiting {span.TotalSeconds} seconds."); + + string relativeUrl = webDirectory.Uri.PathAndQuery; + + if (ex is FtpAuthenticationException ftpAuthenticationException) + { + Logger.Error($"[{context["Processor"]}] Error {ftpAuthenticationException.CompletionCode} {ftpAuthenticationException.Message} retrieving on try {retryCount} for url '{relativeUrl}'. Stopping."); + + if (ftpAuthenticationException.ResponseType == FtpResponseType.PermanentNegativeCompletion) + { + (context["CancellationTokenSource"] as CancellationTokenSource).Cancel(); + return; + } + } + + if (retryCount <= 4) + { + Logger.Warn($"[{context["Processor"]}] Error {ex.Message} retrieving on try {retryCount} for url '{relativeUrl}'. Waiting {span.TotalSeconds:F0} seconds."); + } + else + { + (context["CancellationTokenSource"] as CancellationTokenSource).Cancel(); + } } ); public static Dictionary FtpClients { get; set; } = new Dictionary(); - public static async Task ParseFtpAsync(string processor, WebDirectory webDirectory) + public static async Task ParseFtpAsync(string processor, WebDirectory webDirectory, string username, string password) { + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5)); + Context pollyContext = new Context { - { "Processor", string.Empty }, - { "WebDirectory", webDirectory } + { "Processor", processor }, + { "WebDirectory", webDirectory }, + { "CancellationTokenSource", cancellationTokenSource } }; - return (await RetryPolicy.ExecuteAndCaptureAsync(ctx => ParseFtpInnerAsync(processor, webDirectory), pollyContext)).Result; + return (await RetryPolicyNew.ExecuteAndCaptureAsync(async (context, token) => { return await ParseFtpInnerAsync(processor, webDirectory, username, password, cancellationTokenSource.Token); }, pollyContext, cancellationTokenSource.Token)).Result; } - private static async Task ParseFtpInnerAsync(string processor, WebDirectory webDirectory) + private static async Task ParseFtpInnerAsync(string processor, WebDirectory webDirectory, string username, string password, CancellationToken cancellationToken) { if (!FtpClients.ContainsKey(processor)) { - GetCredentials(webDirectory, out string username, out string password); + if (string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password)) + { + GetCredentials(webDirectory, out string username1, out string password1); + + username = username1; + password = password1; + } FtpClients[processor] = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password) { ConnectTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds, DataConnectionConnectTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds, DataConnectionReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds, - ReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds + ReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds, + EncryptionMode = Enum.Parse(OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_FtpEncryptionMode]) }; - await FtpClients[processor].ConnectAsync(); + FtpClients[processor].ValidateAnyCertificate = true; + + await FtpClients[processor].ConnectAsync(cancellationToken); + + if (!FtpClients[processor].IsConnected) + { + FtpClients.Remove(processor); + throw new Exception("Error connecting to FTP"); + } } // TODO: If anybody knows a better way.. PR! @@ -101,27 +143,52 @@ private static async Task ParseFtpInnerAsync(string processor, Web return webDirectory; } - public static async Task GetFtpServerInfo(WebDirectory webDirectory) + public static async Task GetFtpServerInfo(WebDirectory webDirectory, string username, string password) { + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + + cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5)); + + string processor = "Initalize"; + Context pollyContext = new Context { - { "Processor", string.Empty }, - { "WebDirectory", webDirectory } + { "Processor", processor }, + { "WebDirectory", webDirectory }, + { "CancellationTokenSource", cancellationTokenSource } }; - return (await RetryPolicy.ExecuteAndCaptureAsync(ctx => GetFtpServerInfoInnerAsync(webDirectory), pollyContext)).Result; + return (await RetryPolicyNew.ExecuteAndCaptureAsync(async (context, token) => { return await GetFtpServerInfoInnerAsync(webDirectory, username, password, cancellationTokenSource.Token); }, pollyContext, cancellationTokenSource.Token)).Result; + } + + private static async Task GetFtpServerInfoInnerAsync(WebDirectory webDirectory, string username, string password, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password)) + { + GetCredentials(webDirectory, out string username1, out string password1); + + username = username1; + password = password1; } - private static async Task GetFtpServerInfoInnerAsync(WebDirectory webDirectory) + // Try multiple possible options, the AutoDetect and AutoConnectAsync are not working (reliably) + foreach (FtpEncryptionMode ftpEncryptionMode in Enum.GetValues(typeof(FtpEncryptionMode))) + { + try + { + Logger.Warn($"Try FTP(S) connection with EncryptionMode {ftpEncryptionMode}"); + + FtpClient ftpClient = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password) { - GetCredentials(webDirectory, out string username, out string password); + EncryptionMode = ftpEncryptionMode + }; - FtpClient ftpClient = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password); + ftpClient.ValidateAnyCertificate = true; + await ftpClient.ConnectAsync(cancellationToken); - await ftpClient.ConnectAsync(); + OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_FtpEncryptionMode] = ftpEncryptionMode.ToString(); FtpReply connectReply = ftpClient.LastReply; - CancellationToken cancellationToken = new CancellationToken(); FtpReply helpReply = await ftpClient.ExecuteAsync("HELP", cancellationToken); FtpReply statusReply = await ftpClient.ExecuteAsync("STAT", cancellationToken); @@ -134,6 +201,14 @@ private static async Task GetFtpServerInfoInnerAsync(WebDirectory webDir $"Status response: {statusReply.InfoMessages}{Environment.NewLine}" + $"System response: {systemReply.InfoMessages}{Environment.NewLine}"; } + catch (Exception ex) + { + Logger.Error($"FTP EncryptionMode {ftpEncryptionMode} failed: {ex.Message}"); + } + } + + return null; + } public static async void CloseAll() { diff --git a/OpenDirectoryDownloader/Library.cs b/OpenDirectoryDownloader/Library.cs index d4c4939f..f67e902c 100644 --- a/OpenDirectoryDownloader/Library.cs +++ b/OpenDirectoryDownloader/Library.cs @@ -1,4 +1,4 @@ -using Newtonsoft.Json; +using Newtonsoft.Json; using NLog; using OpenDirectoryDownloader.Helpers; using OpenDirectoryDownloader.Shared; @@ -61,7 +61,7 @@ public static string FixUrl(string url) url = Encoding.UTF8.GetString(data); } - if (!url.Contains("http:") && !url.Contains("https:") && !url.Contains("ftp:")) + if (!url.Contains("http:") && !url.Contains("https:") && !url.Contains("ftp:") && !url.Contains("ftps:")) { url = $"http://{url}"; } diff --git a/OpenDirectoryDownloader/OpenDirectoryIndexer.cs b/OpenDirectoryDownloader/OpenDirectoryIndexer.cs index 79405761..8ee1e30c 100644 --- a/OpenDirectoryDownloader/OpenDirectoryIndexer.cs +++ b/OpenDirectoryDownloader/OpenDirectoryIndexer.cs @@ -169,12 +169,33 @@ public async void StartIndexingAsync() Logger.Warn("Google Drive scanning is limited to 9 directories per second!"); } - if (Session.Root.Uri.Scheme == "ftp") + if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps) { - Logger.Warn("Retrieving FTP software!"); - // TODO: Replace with library? - Logger.Warn(await FtpParser.GetFtpServerInfo(Session.Root)); - //AddProcessedWebDirectory(webDirectory, parsedWebDirectory); + Logger.Warn("Retrieving FTP(S) software!"); + + if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftps) + { + if (Session.Root.Uri.Port == -1) + { + Logger.Warn("Using default port (990) for FTPS"); + + UriBuilder uriBuilder = new UriBuilder(Session.Root.Uri) + { + Port = 990 + }; + + Session.Root.Url = uriBuilder.Uri.ToString(); + } + } + + string serverInfo = await FtpParser.GetFtpServerInfo(Session.Root, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password); + + if (string.IsNullOrWhiteSpace(serverInfo)) + { + serverInfo = "Failed or no server info available."; + } + + Logger.Warn(serverInfo); } TimerStatistics = new System.Timers.Timer @@ -226,7 +247,7 @@ public async void StartIndexingAsync() Console.WriteLine("Finshed indexing"); Logger.Info("Finshed indexing"); - if (Session.Root.Uri.Scheme == "ftp") + if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps) { FtpParser.CloseAll(); } @@ -303,7 +324,7 @@ public async void StartIndexingAsync() { if (Session.TotalFiles > 0) { - if (Session.Root.Uri.Scheme == "https" || Session.Root.Uri.Scheme == "http") + if (Session.Root.Uri.Scheme == Constants.UriScheme.Http || Session.Root.Uri.Scheme == Constants.UriScheme.Https) { try { @@ -451,7 +472,10 @@ private async Task WebDirectoryProcessor(ConcurrentQueue queue, st Logger.Info($"[{name}] Begin processing {webDirectory.Url}"); - if (Session.Root.Uri.Scheme == "ftp") + if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps) + { + WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password); + { WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory); AddProcessedWebDirectory(webDirectory, parsedWebDirectory); @@ -507,11 +531,11 @@ private async Task WebDirectoryProcessor(ConcurrentQueue queue, st { if (webDirectory.ParentDirectory?.Url != null) { - Logger.Warn($"Skipped processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'"); + Logger.Error($"Skipped processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'"); } else { - Logger.Warn($"Skipped processing Url: '{webDirectory.Url}'"); + Logger.Error($"Skipped processing Url: '{webDirectory.Url}'"); Session.Root.Error = true; } } @@ -803,7 +827,7 @@ private void AddProcessedWebDirectory(WebDirectory webDirectory, WebDirectory pa return false; } - return (uri.Scheme != "https" && uri.Scheme != "http" && uri.Scheme != "ftp") || uri.Host != Session.Root.Uri.Host || !SameHostAndDirectory(uri, Session.Root.Uri); + return (uri.Scheme != Constants.UriScheme.Https && uri.Scheme != Constants.UriScheme.Http && uri.Scheme != Constants.UriScheme.Ftp && uri.Scheme != Constants.UriScheme.Ftps) || uri.Host != Session.Root.Uri.Host || !SameHostAndDirectory(uri, Session.Root.Uri); }); foreach (WebFile webFile in webDirectory.Files.Where(f => f.FileSize == -1 || OpenDirectoryIndexerSettings.CommandLineOptions.ExactFileSizes)) diff --git a/OpenDirectoryDownloader/Program.cs b/OpenDirectoryDownloader/Program.cs index 9f66eca2..842751bd 100644 --- a/OpenDirectoryDownloader/Program.cs +++ b/OpenDirectoryDownloader/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using NLog; using System; using System.Collections.Generic; @@ -79,7 +79,7 @@ static async Task Main(string[] args) // FTP // TODO: Make dynamic - if (openDirectoryIndexerSettings.Url?.StartsWith("ftp") == true) + if (openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftp) == true || openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftps) == true) { openDirectoryIndexerSettings.Threads = 6; }