diff --git a/src/OpenDirectoryDownloader/Constants.cs b/src/OpenDirectoryDownloader/Constants.cs index bb37193b..ef33470a 100644 --- a/src/OpenDirectoryDownloader/Constants.cs +++ b/src/OpenDirectoryDownloader/Constants.cs @@ -17,6 +17,8 @@ public class Constants public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss"; public const string Parameters_Password = "PASSWORD"; public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode"; + + public const string GoogleDriveIndexType = "GOOGLEDRIVEINDEXTYPE"; public const long NoFileSize = 0; public const string Root = "ROOT"; diff --git a/src/OpenDirectoryDownloader/DirectoryParser.cs b/src/OpenDirectoryDownloader/DirectoryParser.cs index 538c8b2b..61859336 100644 --- a/src/OpenDirectoryDownloader/DirectoryParser.cs +++ b/src/OpenDirectoryDownloader/DirectoryParser.cs @@ -1,6 +1,8 @@ using AngleSharp.Dom; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; +using Esprima; +using Esprima.Ast; using Newtonsoft.Json; using NLog; using OpenDirectoryDownloader.Helpers; @@ -90,11 +92,13 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri return await MediafireParser.ParseIndex(httpClient, webDirectory); } - if (httpClient is not null) + if (httpClient is not null && !OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.GoogleDriveIndexType)) { + string googleDriveIndexType = null; + foreach (IHtmlScriptElement script in htmlDocument.Scripts.Where(s => s.Source is not null)) { - string googleDriveIndexType = GoogleDriveIndexMapping.GetGoogleDriveIndexType(script.Source); + googleDriveIndexType = GoogleDriveIndexMapping.GetGoogleDriveIndexType(script.Source); if (googleDriveIndexType is null && script.Source.ToLower().Contains("app.min.js")) { @@ -120,29 +124,52 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri } } - if (googleDriveIndexType is not null) + if (googleDriveIndexType is null && script.Source.ToLower().Contains("app.js")) { - if (OpenDirectoryIndexer.Session.MaxThreads != 1) + string appJsSource = await httpClient.GetStringAsync(script.Source); + + JavaScriptParser javaScriptParser = new JavaScriptParser(appJsSource); + Script program = javaScriptParser.ParseScript(); + IEnumerable javaScriptFunctions = program.ChildNodes.OfType(); + FunctionDeclaration gdidecodeFunctionDeclaration = javaScriptFunctions.FirstOrDefault(f => f.ChildNodes.OfType().Any(i => i.Name == "gdidecode")); + + if (gdidecodeFunctionDeclaration is not null) { - OpenDirectoryIndexer.Session.MaxThreads = 1; - Logger.Warn($"Reduce threads to 1 because of Google Drive index"); + googleDriveIndexType = GoogleDriveIndexMapping.BhadooIndex; + break; } } + } - switch (googleDriveIndexType) + if (googleDriveIndexType is not null) + { + OpenDirectoryIndexer.Session.Parameters[Constants.GoogleDriveIndexType] = googleDriveIndexType; + + if (OpenDirectoryIndexer.Session.MaxThreads != 1) { - case GoogleDriveIndexMapping.BhadooIndex: - return await BhadooIndexParser.ParseIndex(htmlDocument, httpClient, webDirectory); - case GoogleDriveIndexMapping.GoIndex: - return await GoIndexParser.ParseIndex(httpClient, webDirectory); - case GoogleDriveIndexMapping.Go2Index: - return await Go2IndexParser.ParseIndex(httpClient, webDirectory); - case GoogleDriveIndexMapping.GdIndex: - return await GdIndexParser.ParseIndex(httpClient, webDirectory, html); + OpenDirectoryIndexer.Session.MaxThreads = 1; + Logger.Warn($"Reduce threads to 1 because of Google Drive index"); } } } + if (OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.GoogleDriveIndexType)) + { + string googleDriveIndexType = OpenDirectoryIndexer.Session.Parameters[Constants.GoogleDriveIndexType]; + + switch (googleDriveIndexType) + { + case GoogleDriveIndexMapping.BhadooIndex: + return await BhadooIndexParser.ParseIndex(htmlDocument, httpClient, webDirectory); + case GoogleDriveIndexMapping.GoIndex: + return await GoIndexParser.ParseIndex(httpClient, webDirectory); + case GoogleDriveIndexMapping.Go2Index: + return await Go2IndexParser.ParseIndex(httpClient, webDirectory); + case GoogleDriveIndexMapping.GdIndex: + return await GdIndexParser.ParseIndex(httpClient, webDirectory, html); + } + } + htmlDocument.QuerySelectorAll("#sidebar").ToList().ForEach(e => e.Remove()); htmlDocument.QuerySelectorAll("nav").ToList().ForEach(e => e.Remove());