Skip to content

Commit

Permalink
- Very experimental support for FTPS
Browse files Browse the repository at this point in the history
  • Loading branch information
@KoalaBear84
KoalaBear84 committed Jul 13, 2020
1 parent 027f2cf commit b26bd0e
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 36 deletions.
9 changes: 9 additions & 0 deletions OpenDirectoryDownloader/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,20 @@ public class Constants
public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss";
public const string Parameters_Password = "PASSWORD";
public const string Parameters_GdIndex_RootId = "GdIndex_RootId";
public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode";

public class UserAgent
{
public const string Curl = "curl/7.55.1";
public const string Chrome = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3800.0 Safari/537.36";
}

public class UriScheme
{
public const string Http = "http";
public const string Https = "https";
public const string Ftp = "ftp";
public const string Ftps = "ftps";
}
}
}
117 changes: 96 additions & 21 deletions OpenDirectoryDownloader/FtpParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,45 +16,87 @@ namespace OpenDirectoryDownloader
public class FtpParser
{
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
private static readonly AsyncRetryPolicy RetryPolicy = Policy
private static readonly Random Jitterer = new Random();
private static readonly AsyncRetryPolicy RetryPolicyNew = Policy
.Handle<Exception>()
.WaitAndRetryAsync(4,
sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
.WaitAndRetryAsync(100,
sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Min(16, Math.Pow(2, retryAttempt))) + TimeSpan.FromMilliseconds(Jitterer.Next(0, 200)),
onRetry: (ex, span, retryCount, context) =>
{
WebDirectory webDirectory = context["WebDirectory"] as WebDirectory;
Logger.Warn($"[{context["Processor"]}] Error {ex.Message} retrieving on try {retryCount} for url '{webDirectory.Url}'. Waiting {span.TotalSeconds} seconds.");

string relativeUrl = webDirectory.Uri.PathAndQuery;

if (ex is FtpAuthenticationException ftpAuthenticationException)
{
Logger.Error($"[{context["Processor"]}] Error {ftpAuthenticationException.CompletionCode} {ftpAuthenticationException.Message} retrieving on try {retryCount} for url '{relativeUrl}'. Stopping.");

if (ftpAuthenticationException.ResponseType == FtpResponseType.PermanentNegativeCompletion)
{
(context["CancellationTokenSource"] as CancellationTokenSource).Cancel();
return;
}
}

if (retryCount <= 4)
{
Logger.Warn($"[{context["Processor"]}] Error {ex.Message} retrieving on try {retryCount} for url '{relativeUrl}'. Waiting {span.TotalSeconds:F0} seconds.");
}
else
{
(context["CancellationTokenSource"] as CancellationTokenSource).Cancel();
}
}
);

public static Dictionary<string, FtpClient> FtpClients { get; set; } = new Dictionary<string, FtpClient>();

public static async Task<WebDirectory> ParseFtpAsync(string processor, WebDirectory webDirectory)
public static async Task<WebDirectory> ParseFtpAsync(string processor, WebDirectory webDirectory, string username, string password)
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();

cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5));

Context pollyContext = new Context
{
{ "Processor", string.Empty },
{ "WebDirectory", webDirectory }
{ "Processor", processor },
{ "WebDirectory", webDirectory },
{ "CancellationTokenSource", cancellationTokenSource }
};

return (await RetryPolicy.ExecuteAndCaptureAsync(ctx => ParseFtpInnerAsync(processor, webDirectory), pollyContext)).Result;
return (await RetryPolicyNew.ExecuteAndCaptureAsync(async (context, token) => { return await ParseFtpInnerAsync(processor, webDirectory, username, password, cancellationTokenSource.Token); }, pollyContext, cancellationTokenSource.Token)).Result;
}

private static async Task<WebDirectory> ParseFtpInnerAsync(string processor, WebDirectory webDirectory)
private static async Task<WebDirectory> ParseFtpInnerAsync(string processor, WebDirectory webDirectory, string username, string password, CancellationToken cancellationToken)
{
if (!FtpClients.ContainsKey(processor))
{
GetCredentials(webDirectory, out string username, out string password);
if (string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password))
{
GetCredentials(webDirectory, out string username1, out string password1);

username = username1;
password = password1;
}

FtpClients[processor] = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password)
{
ConnectTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds,
DataConnectionConnectTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds,
DataConnectionReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds,
ReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds
ReadTimeout = (int)TimeSpan.FromSeconds(30).TotalMilliseconds,
EncryptionMode = Enum.Parse<FtpEncryptionMode>(OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_FtpEncryptionMode])
};

await FtpClients[processor].ConnectAsync();
FtpClients[processor].ValidateAnyCertificate = true;

await FtpClients[processor].ConnectAsync(cancellationToken);

if (!FtpClients[processor].IsConnected)
{
FtpClients.Remove(processor);
throw new Exception("Error connecting to FTP");
}
}

// TODO: If anybody knows a better way.. PR!
Expand Down Expand Up @@ -101,27 +143,52 @@ private static async Task<WebDirectory> ParseFtpInnerAsync(string processor, Web
return webDirectory;
}

public static async Task<string> GetFtpServerInfo(WebDirectory webDirectory)
public static async Task<string> GetFtpServerInfo(WebDirectory webDirectory, string username, string password)
{
CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();

cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5));

string processor = "Initalize";

Context pollyContext = new Context
{
{ "Processor", string.Empty },
{ "WebDirectory", webDirectory }
{ "Processor", processor },
{ "WebDirectory", webDirectory },
{ "CancellationTokenSource", cancellationTokenSource }
};

return (await RetryPolicy.ExecuteAndCaptureAsync(ctx => GetFtpServerInfoInnerAsync(webDirectory), pollyContext)).Result;
return (await RetryPolicyNew.ExecuteAndCaptureAsync(async (context, token) => { return await GetFtpServerInfoInnerAsync(webDirectory, username, password, cancellationTokenSource.Token); }, pollyContext, cancellationTokenSource.Token)).Result;
}

private static async Task<string> GetFtpServerInfoInnerAsync(WebDirectory webDirectory, string username, string password, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(username) && string.IsNullOrWhiteSpace(password))
{
GetCredentials(webDirectory, out string username1, out string password1);

username = username1;
password = password1;
}

private static async Task<string> GetFtpServerInfoInnerAsync(WebDirectory webDirectory)
// Try multiple possible options, the AutoDetect and AutoConnectAsync are not working (reliably)
foreach (FtpEncryptionMode ftpEncryptionMode in Enum.GetValues(typeof(FtpEncryptionMode)))
{
try
{
Logger.Warn($"Try FTP(S) connection with EncryptionMode {ftpEncryptionMode}");

FtpClient ftpClient = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password)
{
GetCredentials(webDirectory, out string username, out string password);
EncryptionMode = ftpEncryptionMode
};

FtpClient ftpClient = new FtpClient(webDirectory.Uri.Host, webDirectory.Uri.Port, username, password);
ftpClient.ValidateAnyCertificate = true;
await ftpClient.ConnectAsync(cancellationToken);

await ftpClient.ConnectAsync();
OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_FtpEncryptionMode] = ftpEncryptionMode.ToString();

FtpReply connectReply = ftpClient.LastReply;
CancellationToken cancellationToken = new CancellationToken();

FtpReply helpReply = await ftpClient.ExecuteAsync("HELP", cancellationToken);
FtpReply statusReply = await ftpClient.ExecuteAsync("STAT", cancellationToken);
Expand All @@ -134,6 +201,14 @@ private static async Task<string> GetFtpServerInfoInnerAsync(WebDirectory webDir
$"Status response: {statusReply.InfoMessages}{Environment.NewLine}" +
$"System response: {systemReply.InfoMessages}{Environment.NewLine}";
}
catch (Exception ex)
{
Logger.Error($"FTP EncryptionMode {ftpEncryptionMode} failed: {ex.Message}");
}
}

return null;
}

public static async void CloseAll()
{
Expand Down
4 changes: 2 additions & 2 deletions OpenDirectoryDownloader/Library.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
using NLog;
using OpenDirectoryDownloader.Helpers;
using OpenDirectoryDownloader.Shared;
Expand Down Expand Up @@ -61,7 +61,7 @@ public static string FixUrl(string url)
url = Encoding.UTF8.GetString(data);
}

if (!url.Contains("http:") && !url.Contains("https:") && !url.Contains("ftp:"))
if (!url.Contains("http:") && !url.Contains("https:") && !url.Contains("ftp:") && !url.Contains("ftps:"))
{
url = $"http://{url}";
}
Expand Down
46 changes: 35 additions & 11 deletions OpenDirectoryDownloader/OpenDirectoryIndexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,12 +169,33 @@ public async void StartIndexingAsync()
Logger.Warn("Google Drive scanning is limited to 9 directories per second!");
}

if (Session.Root.Uri.Scheme == "ftp")
if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
{
Logger.Warn("Retrieving FTP software!");
// TODO: Replace with library?
Logger.Warn(await FtpParser.GetFtpServerInfo(Session.Root));
//AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
Logger.Warn("Retrieving FTP(S) software!");

if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
{
if (Session.Root.Uri.Port == -1)
{
Logger.Warn("Using default port (990) for FTPS");

UriBuilder uriBuilder = new UriBuilder(Session.Root.Uri)
{
Port = 990
};

Session.Root.Url = uriBuilder.Uri.ToString();
}
}

string serverInfo = await FtpParser.GetFtpServerInfo(Session.Root, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password);

if (string.IsNullOrWhiteSpace(serverInfo))
{
serverInfo = "Failed or no server info available.";
}

Logger.Warn(serverInfo);
}

TimerStatistics = new System.Timers.Timer
Expand Down Expand Up @@ -226,7 +247,7 @@ public async void StartIndexingAsync()
Console.WriteLine("Finshed indexing");
Logger.Info("Finshed indexing");

if (Session.Root.Uri.Scheme == "ftp")
if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
{
FtpParser.CloseAll();
}
Expand Down Expand Up @@ -303,7 +324,7 @@ public async void StartIndexingAsync()
{
if (Session.TotalFiles > 0)
{
if (Session.Root.Uri.Scheme == "https" || Session.Root.Uri.Scheme == "http")
if (Session.Root.Uri.Scheme == Constants.UriScheme.Http || Session.Root.Uri.Scheme == Constants.UriScheme.Https)
{
try
{
Expand Down Expand Up @@ -451,7 +472,10 @@ private async Task WebDirectoryProcessor(ConcurrentQueue<WebDirectory> queue, st

Logger.Info($"[{name}] Begin processing {webDirectory.Url}");

if (Session.Root.Uri.Scheme == "ftp")
if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
{
WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory, OpenDirectoryIndexerSettings.Username, OpenDirectoryIndexerSettings.Password);

{
WebDirectory parsedWebDirectory = await FtpParser.ParseFtpAsync(name, webDirectory);
AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
Expand Down Expand Up @@ -507,11 +531,11 @@ private async Task WebDirectoryProcessor(ConcurrentQueue<WebDirectory> queue, st
{
if (webDirectory.ParentDirectory?.Url != null)
{
Logger.Warn($"Skipped processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'");
Logger.Error($"Skipped processing Url: '{webDirectory.Url}' from parent '{webDirectory.ParentDirectory.Url}'");
}
else
{
Logger.Warn($"Skipped processing Url: '{webDirectory.Url}'");
Logger.Error($"Skipped processing Url: '{webDirectory.Url}'");
Session.Root.Error = true;
}
}
Expand Down Expand Up @@ -803,7 +827,7 @@ private void AddProcessedWebDirectory(WebDirectory webDirectory, WebDirectory pa
return false;
}

return (uri.Scheme != "https" && uri.Scheme != "http" && uri.Scheme != "ftp") || uri.Host != Session.Root.Uri.Host || !SameHostAndDirectory(uri, Session.Root.Uri);
return (uri.Scheme != Constants.UriScheme.Https && uri.Scheme != Constants.UriScheme.Http && uri.Scheme != Constants.UriScheme.Ftp && uri.Scheme != Constants.UriScheme.Ftps) || uri.Host != Session.Root.Uri.Host || !SameHostAndDirectory(uri, Session.Root.Uri);
});

foreach (WebFile webFile in webDirectory.Files.Where(f => f.FileSize == -1 || OpenDirectoryIndexerSettings.CommandLineOptions.ExactFileSizes))
Expand Down
4 changes: 2 additions & 2 deletions OpenDirectoryDownloader/Program.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using CommandLine;
using CommandLine;
using NLog;
using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -79,7 +79,7 @@ static async Task<int> Main(string[] args)

// FTP
// TODO: Make dynamic
if (openDirectoryIndexerSettings.Url?.StartsWith("ftp") == true)
if (openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftp) == true || openDirectoryIndexerSettings.Url?.StartsWith(Constants.UriScheme.Ftps) == true)
{
openDirectoryIndexerSettings.Threads = 6;
}
Expand Down

0 comments on commit b26bd0e

Please sign in to comment.