-
-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
f8c77f8
commit 9695640
Showing
4 changed files
with
296 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
src/OpenDirectoryDownloader/Site/CrushFtp/CrushFtpParser.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| using NLog; | ||
| using OpenDirectoryDownloader.Shared; | ||
| using OpenDirectoryDownloader.Shared.Models; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Net.Http; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace OpenDirectoryDownloader.Site.CrushFtp; | ||
|
|
||
| public static class CrushFtpParser | ||
| { | ||
| private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); | ||
| private const string Parser = "CrushFTP"; | ||
| private static string Authentication; | ||
| //private static string Username; | ||
| private static string FunctionUrl; | ||
| private static readonly Random Random = new(); | ||
| private static bool WarningShown = false; | ||
| private static readonly RateLimiter RateLimiter = new(1, TimeSpan.FromSeconds(1), margin: 1); | ||
|
|
||
| public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory) | ||
| { | ||
| if (!WarningShown) | ||
| { | ||
| WarningShown = true; | ||
|
|
||
| Logger.Warn($"CrushFTP scanning is limited to {RateLimiter.MaxRequestsPerTimeSpan} directories per {RateLimiter.TimeSpan.TotalSeconds:F1} second(s)!"); | ||
| } | ||
|
|
||
| try | ||
| { | ||
| if (string.IsNullOrWhiteSpace(Authentication)) | ||
| { | ||
| Authentication = OpenDirectoryIndexer.CookieContainer.GetAllCookies().FirstOrDefault(c => c.Name == "currentAuth")?.Value; | ||
| FunctionUrl = $"{webDirectory.Uri.GetComponents(UriComponents.Scheme | UriComponents.Host | UriComponents.Port, UriFormat.UriEscaped)}/WebInterface/function/"; | ||
| } | ||
|
|
||
| webDirectory = await ScanAsync(httpClient, webDirectory); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); | ||
| webDirectory.Error = true; | ||
|
|
||
| throw; | ||
| } | ||
|
|
||
| return webDirectory; | ||
| } | ||
|
|
||
| private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory) | ||
| { | ||
| Logger.Debug($"Retrieving listings for {webDirectory.Uri}"); | ||
|
|
||
| webDirectory.Parser = Parser; | ||
|
|
||
| try | ||
| { | ||
| await RateLimiter.RateLimit(); | ||
|
|
||
| Dictionary<string, string> postValues = new() | ||
| { | ||
| { "command", "getXMLListing" }, | ||
| { "format", "JSONOBJ" }, | ||
| { "path", $"{Uri.EscapeDataString(webDirectory.Uri.LocalPath)}%2F" }, | ||
| { "random", Random.NextDouble().ToString() }, | ||
| { "c2f", Authentication } | ||
| }; | ||
|
|
||
| HttpResponseMessage httpResponseMessage = await httpClient.PostAsync(FunctionUrl, new FormUrlEncodedContent(postValues)); | ||
|
|
||
| httpResponseMessage.EnsureSuccessStatusCode(); | ||
|
|
||
| string response = await httpResponseMessage.Content.ReadAsStringAsync(); | ||
|
|
||
| CrushFtpResult crushFtpResult = CrushFtpResult.FromJson(response); | ||
|
|
||
| foreach (Listing listing in crushFtpResult.Listing) | ||
| { | ||
| if (listing.Type == "DIR") | ||
| { | ||
| webDirectory.Subdirectories.Add(new WebDirectory(webDirectory) | ||
| { | ||
| Parser = Parser, | ||
| Url = new Uri(webDirectory.Uri, listing.HrefPath).ToString(), | ||
| Name = listing.Name | ||
| }); | ||
| } | ||
| else | ||
| { | ||
| webDirectory.Files.Add(new WebFile | ||
| { | ||
| Url = new Uri(webDirectory.Uri, listing.HrefPath).ToString(), | ||
| FileName = Path.GetFileName(listing.HrefPath), | ||
| FileSize = listing.Size | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); | ||
| webDirectory.Error = true; | ||
|
|
||
| throw; | ||
| } | ||
|
|
||
| return webDirectory; | ||
| } | ||
| } |
162 changes: 162 additions & 0 deletions
162
src/OpenDirectoryDownloader/Site/CrushFtp/CrushFtpResult.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,162 @@ | ||
| // <auto-generated /> | ||
| // | ||
| // To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: | ||
| // | ||
| // using QuickType; | ||
| // | ||
| // var crushFtpResponse = CrushFtpResponse.FromJson(jsonString); | ||
|
|
||
| namespace OpenDirectoryDownloader.Site.CrushFtp; | ||
|
|
||
| using System.Globalization; | ||
| using Newtonsoft.Json; | ||
| using Newtonsoft.Json.Converters; | ||
|
|
||
| public partial class CrushFtpResult | ||
| { | ||
| [JsonProperty("privs")] | ||
| public string Privs { get; set; } | ||
|
|
||
| [JsonProperty("comment")] | ||
| public string Comment { get; set; } | ||
|
|
||
| [JsonProperty("path")] | ||
| public string Path { get; set; } | ||
|
|
||
| [JsonProperty("defaultStrings")] | ||
| public string DefaultStrings { get; set; } | ||
|
|
||
| [JsonProperty("site")] | ||
| public string Site { get; set; } | ||
|
|
||
| [JsonProperty("quota")] | ||
| public string Quota { get; set; } | ||
|
|
||
| [JsonProperty("quota_bytes")] | ||
| public string QuotaBytes { get; set; } | ||
|
|
||
| [JsonProperty("bytes_sent")] | ||
| public long BytesSent { get; set; } | ||
|
|
||
| [JsonProperty("bytes_received")] | ||
| public long BytesReceived { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount_day")] | ||
| public long MaxUploadAmountDay { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount_month")] | ||
| public long MaxUploadAmountMonth { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount")] | ||
| public long MaxUploadAmount { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount_available")] | ||
| public long MaxUploadAmountAvailable { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount_day_available")] | ||
| public string MaxUploadAmountDayAvailable { get; set; } | ||
|
|
||
| [JsonProperty("max_upload_amount_month_available")] | ||
| public string MaxUploadAmountMonthAvailable { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount")] | ||
| public long MaxDownloadAmount { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount_day")] | ||
| public long MaxDownloadAmountDay { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount_month")] | ||
| public long MaxDownloadAmountMonth { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount_available")] | ||
| public long MaxDownloadAmountAvailable { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount_day_available")] | ||
| public string MaxDownloadAmountDayAvailable { get; set; } | ||
|
|
||
| [JsonProperty("max_download_amount_month_available")] | ||
| public string MaxDownloadAmountMonthAvailable { get; set; } | ||
|
|
||
| [JsonProperty("listing")] | ||
| public Listing[] Listing { get; set; } | ||
| } | ||
|
|
||
| public partial class Listing | ||
| { | ||
| [JsonProperty("name")] | ||
| public string Name { get; set; } | ||
|
|
||
| [JsonProperty("dir")] | ||
| public string Dir { get; set; } | ||
|
|
||
| [JsonProperty("type")] | ||
| public string Type { get; set; } | ||
|
|
||
| [JsonProperty("root_dir")] | ||
| public string RootDir { get; set; } | ||
|
|
||
| [JsonProperty("href_path")] | ||
| public string HrefPath { get; set; } | ||
|
|
||
| [JsonProperty("size")] | ||
| public long Size { get; set; } | ||
|
|
||
| [JsonProperty("modified")] | ||
| public string Modified { get; set; } | ||
|
|
||
| [JsonProperty("created")] | ||
| public string Created { get; set; } | ||
|
|
||
| [JsonProperty("owner")] | ||
| public string Owner { get; set; } | ||
|
|
||
| [JsonProperty("group")] | ||
| public string Group { get; set; } | ||
|
|
||
| [JsonProperty("permissionsNum")] | ||
| public string PermissionsNum { get; set; } | ||
|
|
||
| [JsonProperty("keywords")] | ||
| public string Keywords { get; set; } | ||
|
|
||
| [JsonProperty("permissions")] | ||
| public string Permissions { get; set; } | ||
|
|
||
| [JsonProperty("num_items")] | ||
| public long NumItems { get; set; } | ||
|
|
||
| [JsonProperty("preview")] | ||
| public long Preview { get; set; } | ||
|
|
||
| [JsonProperty("dateFormatted")] | ||
| public string DateFormatted { get; set; } | ||
|
|
||
| [JsonProperty("createdDateFormatted")] | ||
| public string CreatedDateFormatted { get; set; } | ||
|
|
||
| [JsonProperty("sizeFormatted")] | ||
| public string SizeFormatted { get; set; } | ||
| } | ||
|
|
||
| public partial class CrushFtpResult | ||
| { | ||
| public static CrushFtpResult FromJson(string json) => JsonConvert.DeserializeObject<CrushFtpResult>(json, Converter.Settings); | ||
| } | ||
|
|
||
| public static class Serialize | ||
| { | ||
| public static string ToJson(this CrushFtpResult self) => JsonConvert.SerializeObject(self, Converter.Settings); | ||
| } | ||
|
|
||
| internal static class Converter | ||
| { | ||
| public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings | ||
| { | ||
| MetadataPropertyHandling = MetadataPropertyHandling.Ignore, | ||
| DateParseHandling = DateParseHandling.None, | ||
| Converters = | ||
| { | ||
| new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } | ||
| }, | ||
| }; | ||
| } |