From 7361015bfdf004454b240bbf2f41d749a0d0daa0 Mon Sep 17 00:00:00 2001 From: KoalaBear Date: Thu, 31 Oct 2024 07:11:11 +0100 Subject: [PATCH] - Semantical changes - Decrease nesting --- ...OpenDirectoryDownloader.GoogleDrive.csproj | 2 +- .../OpenDirectoryDownloader.Tests.csproj | 6 +- src/OpenDirectoryDownloader/BrowserContext.cs | 52 +- src/OpenDirectoryDownloader/Command.cs | 44 +- .../DirectoryParser.cs | 2700 +++++++++-------- src/OpenDirectoryDownloader/FtpParser.cs | 59 +- .../Helpers/FileSizeHelper.cs | 57 +- src/OpenDirectoryDownloader/Library.cs | 69 +- .../OpenDirectoryDownloader.csproj | 11 +- .../OpenDirectoryIndexer.cs | 302 +- src/OpenDirectoryDownloader/Program.cs | 10 +- .../Site/Copyparty/CopypartyParser.cs | 52 +- src/OpenDirectoryDownloader/Statistics.cs | 49 +- .../UrlEncodingParser.cs | 20 +- 14 files changed, 1852 insertions(+), 1581 deletions(-) diff --git a/src/OpenDirectoryDownloader.GoogleDrive/OpenDirectoryDownloader.GoogleDrive.csproj b/src/OpenDirectoryDownloader.GoogleDrive/OpenDirectoryDownloader.GoogleDrive.csproj index 99c2d9e3..2093261f 100644 --- a/src/OpenDirectoryDownloader.GoogleDrive/OpenDirectoryDownloader.GoogleDrive.csproj +++ b/src/OpenDirectoryDownloader.GoogleDrive/OpenDirectoryDownloader.GoogleDrive.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/OpenDirectoryDownloader.Tests/OpenDirectoryDownloader.Tests.csproj b/src/OpenDirectoryDownloader.Tests/OpenDirectoryDownloader.Tests.csproj index 78bfdb52..8351f0b3 100644 --- a/src/OpenDirectoryDownloader.Tests/OpenDirectoryDownloader.Tests.csproj +++ b/src/OpenDirectoryDownloader.Tests/OpenDirectoryDownloader.Tests.csproj @@ -12,9 +12,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers diff --git a/src/OpenDirectoryDownloader/BrowserContext.cs b/src/OpenDirectoryDownloader/BrowserContext.cs index 6e7b2e28..fbb90c9f 100644 --- a/src/OpenDirectoryDownloader/BrowserContext.cs +++ b/src/OpenDirectoryDownloader/BrowserContext.cs @@ -142,7 +142,7 @@ public async Task InitializeAsync() try { - if (!browserFetcher.GetInstalledBrowsers().Any(x => x.BuildId == Chrome.DefaultBuildId)) + if (browserFetcher.GetInstalledBrowsers().All(x => x.BuildId != Chrome.DefaultBuildId)) { Program.Logger.Warning("Downloading browser... First time it can take a while, depending on your internet connection."); InstalledBrowser installedBrowser = await browserFetcher.DownloadAsync(Chrome.DefaultBuildId); @@ -164,9 +164,9 @@ public async Task InitializeAsync() Browser = await puppeteerExtra.LaunchAsync(new LaunchOptions { Headless = false, - Args = new[] { "--no-sandbox", "--disable-setuid-sandbox", $"--user-agent=\"{Constants.UserAgent.Chrome}\"" }, + Args = ["--no-sandbox", "--disable-setuid-sandbox", $"--user-agent=\"{Constants.UserAgent.Chrome}\""], DefaultViewport = null, - IgnoreHTTPSErrors = true + AcceptInsecureCerts = true }); Program.Logger.Information("Started browser with PID {processId}", Browser.Process.Id); @@ -311,7 +311,7 @@ private void Page_Error(object sender, ErrorEventArgs e) WriteDebugInfo($"Page_Error: {e.Error}"); } - private async void Page_FrameAttached(object sender, FrameEventArgs e) + private void Page_FrameAttached(object sender, FrameEventArgs e) { WriteDebugInfo($"Page_FrameAttached: {e.Frame.Url}"); } @@ -379,23 +379,29 @@ private void Page_Response(object sender, ResponseCreatedEventArgs e) string theCookie = cookieHeader.Split('\n').FirstOrDefault(cookie => cookie.StartsWith(CloudflareClearanceKey)); - if (theCookie != null) + if (theCookie == null) { - CookieContainer.SetCookies(new Uri(baseUrl), theCookie); + return; + } - if (CloudFlare) - { - Cookie cloudflareClearance = CookieContainer.GetCookies(new Uri(baseUrl)).FirstOrDefault(c => c.Name == CloudflareClearanceKey); + CookieContainer.SetCookies(new Uri(baseUrl), theCookie); - if (cloudflareClearance != null) - { - WriteDebugInfo($"Cloudflare clearance cookie found: {cloudflareClearance.Value}"); + if (!CloudFlare) + { + return; + } - OK = true; - CancellationTokenSource.Cancel(); - } - } + Cookie cloudflareClearance = CookieContainer.GetCookies(new Uri(baseUrl)).FirstOrDefault(c => c.Name == CloudflareClearanceKey); + + if (cloudflareClearance == null) + { + return; } + + WriteDebugInfo($"Cloudflare clearance cookie found: {cloudflareClearance.Value}"); + + OK = true; + CancellationTokenSource.Cancel(); } else { @@ -425,15 +431,17 @@ private void CheckCloudflareCookie() CookieParam cloudflareClearanceCookie = cookieParams.FirstOrDefault(cookie => cookie.Name.StartsWith(CloudflareClearanceKey)); - if (cloudflareClearanceCookie is not null) + if (cloudflareClearanceCookie is null) { - WriteDebugInfo($"Cloudflare clearance cookie found: {cloudflareClearanceCookie.Value}"); + return; + } - AddCookiesToContainer(CookieContainer, cookieParams); + WriteDebugInfo($"Cloudflare clearance cookie found: {cloudflareClearanceCookie.Value}"); - OK = true; - CancellationTokenSource.Cancel(); - } + AddCookiesToContainer(CookieContainer, cookieParams); + + OK = true; + CancellationTokenSource.Cancel(); } catch { diff --git a/src/OpenDirectoryDownloader/Command.cs b/src/OpenDirectoryDownloader/Command.cs index 949ddafa..bcc7d003 100644 --- a/src/OpenDirectoryDownloader/Command.cs +++ b/src/OpenDirectoryDownloader/Command.cs @@ -16,11 +16,13 @@ public class Command /// internal static void SetConsoleProperties() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - Console.WindowWidth = Math.Min(170, Console.LargestWindowWidth); - Console.WindowHeight = Math.Min(34, Console.LargestWindowHeight); + return; } + + Console.WindowWidth = Math.Min(170, Console.LargestWindowWidth); + Console.WindowHeight = Math.Min(34, Console.LargestWindowHeight); } internal static void ShowInfoAndCommands() @@ -253,28 +255,30 @@ private static void SetClipboard(string value) { } - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - try + return; + } + + try + { + Process clipboardExecutable = new() { - Process clipboardExecutable = new() + StartInfo = new ProcessStartInfo { - StartInfo = new ProcessStartInfo - { - RedirectStandardInput = true, - FileName = @"wl-copy", - } - }; + RedirectStandardInput = true, + FileName = @"wl-copy", + } + }; - clipboardExecutable.Start(); - clipboardExecutable.StandardInput.Write(value); - clipboardExecutable.StandardInput.Close(); + clipboardExecutable.Start(); + clipboardExecutable.StandardInput.Write(value); + clipboardExecutable.StandardInput.Close(); - return; - } - catch - { - } + return; + } + catch + { } } } diff --git a/src/OpenDirectoryDownloader/DirectoryParser.cs b/src/OpenDirectoryDownloader/DirectoryParser.cs index 4039c0ff..899a0faf 100644 --- a/src/OpenDirectoryDownloader/DirectoryParser.cs +++ b/src/OpenDirectoryDownloader/DirectoryParser.cs @@ -41,11 +41,14 @@ public static class DirectoryParser /// Base url /// Html to parse /// WebDirectory object containing current directory index - public static async Task ParseHtml(WebDirectory webDirectory, string html, HttpClient httpClient = null, SocketsHttpHandler socketsHttpHandler = null, HttpResponseMessage httpResponseMessage = null, bool checkParents = true) + public static async Task ParseHtml(WebDirectory webDirectory, string html, + HttpClient httpClient = null, SocketsHttpHandler socketsHttpHandler = null, + HttpResponseMessage httpResponseMessage = null, bool checkParents = true) { string baseUrl = webDirectory.Url; - if (!baseUrl.EndsWith("/") && string.IsNullOrWhiteSpace(webDirectory.Uri.Query) && string.IsNullOrWhiteSpace(Path.GetExtension(baseUrl))) + if (!baseUrl.EndsWith("/") && string.IsNullOrWhiteSpace(webDirectory.Uri.Query) && + string.IsNullOrWhiteSpace(Path.GetExtension(baseUrl))) { baseUrl += "/"; } @@ -53,7 +56,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri WebDirectory parsedWebDirectory = new(webDirectory.ParentDirectory) { Url = baseUrl, - Name = WebUtility.UrlDecode(Path.GetDirectoryName(new Uri(baseUrl).Segments.LastOrDefault())) ?? Constants.Root + Name = WebUtility.UrlDecode(Path.GetDirectoryName(new Uri(baseUrl).Segments.LastOrDefault())) ?? + Constants.Root }; try @@ -108,12 +112,14 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (baseUrlRegexMatch.Success) { - return await FileBrowserParser.ParseIndex(baseUrl, httpClient, parsedWebDirectory, htmlDocument, html); + return await FileBrowserParser.ParseIndex(baseUrl, httpClient, parsedWebDirectory, htmlDocument, + html); } } } - if (httpClient is not null && !OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.GoogleDriveIndexType)) + if (httpClient is not null && + !OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.GoogleDriveIndexType)) { string googleDriveIndexType = null; @@ -121,17 +127,23 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri { googleDriveIndexType = GoogleDriveIndexMapping.GetGoogleDriveIndexType(script.Source); - if (googleDriveIndexType is null && script.Source.ToLowerInvariant().Contains("app.min.js")) + if (googleDriveIndexType is null && + script.Source.Contains("app.min.js", StringComparison.InvariantCultureIgnoreCase)) { - Program.Logger.Warning("Checking/downloading javascript for sourcemaps: {scriptUrl}", script.Source); - string sourceMapUrl = await Library.GetSourceMapUrlFromJavaScriptAsync(httpClient, script.Source); + Program.Logger.Warning("Checking/downloading javascript for sourcemaps: {scriptUrl}", + script.Source); + string sourceMapUrl = + await Library.GetSourceMapUrlFromJavaScriptAsync(httpClient, script.Source); if (!string.IsNullOrWhiteSpace(sourceMapUrl)) { string fullSourceMapUrl = new Uri(new Uri(script.Source), sourceMapUrl).ToString(); - Program.Logger.Warning("Checking/downloading sourcemap for known Google Drive index: {sourceMapUrl}", fullSourceMapUrl); + Program.Logger.Warning( + "Checking/downloading sourcemap for known Google Drive index: {sourceMapUrl}", + fullSourceMapUrl); - IAsyncEnumerable sources = Library.GetSourcesFromSourceMapAsync(httpClient, fullSourceMapUrl); + IAsyncEnumerable sources = + Library.GetSourcesFromSourceMapAsync(httpClient, fullSourceMapUrl); await foreach (string source in sources) { @@ -145,21 +157,24 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri } } - if (googleDriveIndexType is null && script.Source.ToLowerInvariant().Contains("app.js")) + if (googleDriveIndexType is null && + script.Source.Contains("app.js", StringComparison.InvariantCultureIgnoreCase)) { string scriptUrl = script.Source; if (Uri.IsWellFormedUriString(scriptUrl, UriKind.Relative)) { - scriptUrl = (new Uri(OpenDirectoryIndexer.Session.Root.Uri, scriptUrl)).ToString(); + scriptUrl = new Uri(OpenDirectoryIndexer.Session.Root.Uri, scriptUrl).ToString(); } string appJsSource = await httpClient.GetStringAsync(scriptUrl); Parser javaScriptParser = new(); Script program = javaScriptParser.ParseScript(appJsSource); - IEnumerable javaScriptFunctions = program.ChildNodes.OfType(); - FunctionDeclaration gdidecodeFunctionDeclaration = javaScriptFunctions.FirstOrDefault(f => f.ChildNodes.OfType().Any(i => i.Name == "gdidecode")); + IEnumerable javaScriptFunctions = + program.ChildNodes.OfType(); + FunctionDeclaration gdidecodeFunctionDeclaration = javaScriptFunctions.FirstOrDefault(f => + f.ChildNodes.OfType().Any(i => i.Name == "gdidecode")); if (gdidecodeFunctionDeclaration is not null) { @@ -186,7 +201,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri } } - if (OpenDirectoryIndexer.Session is not null && OpenDirectoryIndexer.Session.Parameters.TryGetValue(Constants.GoogleDriveIndexType, out string value)) + if (OpenDirectoryIndexer.Session is not null && + OpenDirectoryIndexer.Session.Parameters.TryGetValue(Constants.GoogleDriveIndexType, out string value)) { string googleDriveIndexType = value; @@ -228,11 +244,13 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri // This is already handled by normal parsers } - IHtmlCollection directoryListingDotComlistItems = htmlDocument.QuerySelectorAll("#directory-listing li, .directory-listing li"); + IHtmlCollection directoryListingDotComlistItems = + htmlDocument.QuerySelectorAll("#directory-listing li, .directory-listing li"); if (directoryListingDotComlistItems.Any()) { - return ParseDirectoryListingDoctComDirectoryListing(baseUrl, parsedWebDirectory, directoryListingDotComlistItems, checkParents); + return ParseDirectoryListingDoctComDirectoryListing(baseUrl, parsedWebDirectory, + directoryListingDotComlistItems, checkParents); } IHtmlCollection h5aiTableRows = htmlDocument.QuerySelectorAll("#fallback table tr"); @@ -256,7 +274,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (pureTableRows.Any()) { - return ParsePureDirectoryListing(ref baseUrl, parsedWebDirectory, htmlDocument, pureTableRows, checkParents); + return ParsePureDirectoryListing(ref baseUrl, parsedWebDirectory, htmlDocument, pureTableRows, + checkParents); } // Remove it after ParsePureDirectoryListing (.breadcrumb is used in it) @@ -306,7 +325,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri WebDirectory parsedJavaScriptDrawn = ParseJavaScriptDrawn(baseUrl, parsedWebDirectory, html); - if (parsedJavaScriptDrawn.ParsedSuccessfully && (parsedJavaScriptDrawn.Files.Any() || parsedJavaScriptDrawn.Subdirectories.Any())) + if (parsedJavaScriptDrawn.ParsedSuccessfully && (parsedJavaScriptDrawn.Files.Count != 0 || + parsedJavaScriptDrawn.Subdirectories.Count != 0)) { return parsedJavaScriptDrawn; } @@ -315,7 +335,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (listItems.Any()) { - WebDirectory result = ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); + WebDirectory result = + ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); if (result.ParsedSuccessfully || result.Error) { @@ -346,7 +367,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (materialDesignListItems.Any()) { - return ParseMaterialDesignListItemsDirectoryListing(baseUrl, parsedWebDirectory, materialDesignListItems, checkParents); + return ParseMaterialDesignListItemsDirectoryListing(baseUrl, parsedWebDirectory, + materialDesignListItems, checkParents); } if (htmlDocument.QuerySelectorAll("#content ul#file-list li").Length == 2) @@ -358,7 +380,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (listItems.Any()) { - WebDirectory result = ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); + WebDirectory result = + ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); if (result.ParsedSuccessfully || result.Error) { @@ -370,7 +393,8 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri if (listItems.Any()) { - WebDirectory result = ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); + WebDirectory result = + ParseListItemsDirectoryListing(baseUrl, parsedWebDirectory, listItems, checkParents); if (result.ParsedSuccessfully || result.Error) { @@ -386,14 +410,18 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri parsedWebDirectory = ParseLinksDirectoryListing(baseUrl, parsedWebDirectory, links, checkParents); } - parsedWebDirectory = await ParseDirectoryListingModel01(baseUrl, parsedWebDirectory, htmlDocument, httpClient); + parsedWebDirectory = + await ParseDirectoryListingModel01(baseUrl, parsedWebDirectory, htmlDocument, httpClient); CheckParsedResults(parsedWebDirectory, baseUrl, checkParents); - if (parsedWebDirectory.Subdirectories.Count == 0 && parsedWebDirectory.Files.Count == 0 && !OpenDirectoryIndexer.Session?.ProcessedBrowserUrls.Contains(webDirectory.Url) == false && - htmlDocument.QuerySelector("noscript") != null && htmlDocument.QuerySelector("script") != null) + if (parsedWebDirectory.Subdirectories.Count == 0 && parsedWebDirectory.Files.Count == 0 && + !OpenDirectoryIndexer.Session?.ProcessedBrowserUrls.Contains(webDirectory.Url) == false && + htmlDocument.QuerySelector("noscript") != null && htmlDocument.QuerySelector("script") != null) { - Program.Logger.Warning("No directories and files found on {url}, but did find a