-
-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
60b7ed7
commit b6178cf
Showing
5 changed files
with
307 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
134 changes: 134 additions & 0 deletions
134
src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOListingResult.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| // <auto-generated /> | ||
| // | ||
| // To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: | ||
| // | ||
| // using QuickType; | ||
| // | ||
| // var goFileListing = GoFileIOResult.FromJson(jsonString); | ||
|
|
||
| namespace OpenDirectoryDownloader.Site.GoFileIO | ||
| { | ||
| using System; | ||
| using System.Collections.Generic; | ||
|
|
||
| using System.Globalization; | ||
| using Newtonsoft.Json; | ||
| using Newtonsoft.Json.Converters; | ||
|
|
||
| public partial class GoFileIOListingResult | ||
| { | ||
| [JsonProperty("status")] | ||
| public string Status { get; set; } | ||
|
|
||
| [JsonProperty("data")] | ||
| public Data Data { get; set; } | ||
| } | ||
|
|
||
| public partial class Data | ||
| { | ||
| // Only for /createAccount | ||
| [JsonProperty("token")] | ||
| public string Token { get; set; } | ||
|
|
||
| [JsonProperty("isOwner")] | ||
| public bool IsOwner { get; set; } | ||
|
|
||
| [JsonProperty("id")] | ||
| public Guid Id { get; set; } | ||
|
|
||
| [JsonProperty("type")] | ||
| public string Type { get; set; } | ||
|
|
||
| [JsonProperty("name")] | ||
| public string Name { get; set; } | ||
|
|
||
| [JsonProperty("parentFolder")] | ||
| public Guid ParentFolder { get; set; } | ||
|
|
||
| [JsonProperty("code")] | ||
| public string Code { get; set; } | ||
|
|
||
| [JsonProperty("createTime")] | ||
| public long CreateTime { get; set; } | ||
|
|
||
| [JsonProperty("public")] | ||
| public bool Public { get; set; } | ||
|
|
||
| [JsonProperty("description")] | ||
| public string Description { get; set; } | ||
|
|
||
| [JsonProperty("childs")] | ||
| public Guid[] Childs { get; set; } | ||
|
|
||
| [JsonProperty("totalDownloadCount")] | ||
| public long TotalDownloadCount { get; set; } | ||
|
|
||
| [JsonProperty("totalSize")] | ||
| public long TotalSize { get; set; } | ||
|
|
||
| [JsonProperty("contents")] | ||
| public Dictionary<string, Content> Contents { get; set; } | ||
| } | ||
|
|
||
| public partial class Content | ||
| { | ||
| [JsonProperty("id")] | ||
| public string Id { get; set; } | ||
|
|
||
| [JsonProperty("type")] | ||
| public string Type { get; set; } | ||
|
|
||
| [JsonProperty("name")] | ||
| public string Name { get; set; } | ||
|
|
||
| [JsonProperty("parentFolder")] | ||
| public Guid ParentFolder { get; set; } | ||
|
|
||
| [JsonProperty("createTime")] | ||
| public long CreateTime { get; set; } | ||
|
|
||
| [JsonProperty("size")] | ||
| public long Size { get; set; } | ||
|
|
||
| [JsonProperty("downloadCount")] | ||
| public long DownloadCount { get; set; } | ||
|
|
||
| [JsonProperty("md5")] | ||
| public string Md5 { get; set; } | ||
|
|
||
| [JsonProperty("mimetype")] | ||
| public string Mimetype { get; set; } | ||
|
|
||
| [JsonProperty("serverChoosen")] | ||
| public string ServerChoosen { get; set; } | ||
|
|
||
| [JsonProperty("directLink")] | ||
| public Uri DirectLink { get; set; } | ||
|
|
||
| [JsonProperty("link")] | ||
| public Uri Link { get; set; } | ||
| } | ||
|
|
||
| public partial class GoFileIOListingResult | ||
| { | ||
| public static GoFileIOListingResult FromJson(string json) => JsonConvert.DeserializeObject<GoFileIOListingResult>(json, Converter.Settings); | ||
| } | ||
|
|
||
| public static class Serialize | ||
| { | ||
| public static string ToJson(this GoFileIOListingResult self) => JsonConvert.SerializeObject(self, Converter.Settings); | ||
| } | ||
|
|
||
| internal static class Converter | ||
| { | ||
| public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings | ||
| { | ||
| MetadataPropertyHandling = MetadataPropertyHandling.Ignore, | ||
| DateParseHandling = DateParseHandling.None, | ||
| Converters = | ||
| { | ||
| new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } | ||
| }, | ||
| }; | ||
| } | ||
| } |
153 changes: 153 additions & 0 deletions
153
src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOParser.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| using NLog; | ||
| using OpenDirectoryDownloader.Shared.Models; | ||
| using System; | ||
| using System.Net.Http; | ||
| using System.Text.RegularExpressions; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace OpenDirectoryDownloader.Site.GoFileIO; | ||
|
|
||
| /// <summary> | ||
| /// Similar to GoFile.IO | ||
| /// </summary> | ||
| public static class GoFileIOParser | ||
| { | ||
| private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); | ||
| private static readonly Regex FolderHashRegex = new Regex(@".*?\/d\/(?<FolderHash>.*)"); | ||
| private const string Parser = "GoFileIO"; | ||
| private const string StatusOK = "ok"; | ||
| private const string ApiBaseAddress = "https://api.gofile.io"; | ||
|
|
||
| public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory) | ||
| { | ||
| try | ||
| { | ||
| string driveHash = GetDriveHash(webDirectory); | ||
|
|
||
| if (!OpenDirectoryIndexer.Session.Parameters.ContainsKey(Constants.Parameters_GoFileIOAccountToken)) | ||
| { | ||
| Console.WriteLine($"{Parser} creating temporary account."); | ||
| Logger.Info($"{Parser} creating temporary account."); | ||
|
|
||
| HttpResponseMessage httpResponseMessage = await httpClient.GetAsync($"{ApiBaseAddress}/createAccount"); | ||
|
|
||
| if (httpResponseMessage.IsSuccessStatusCode) | ||
| { | ||
| string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); | ||
|
|
||
| GoFileIOListingResult response = GoFileIOListingResult.FromJson(responseJson); | ||
|
|
||
| if (response.Status != StatusOK) | ||
| { | ||
| throw new Exception($"Error creating account: {response.Status}"); | ||
| } | ||
|
|
||
| OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_GoFileIOAccountToken] = response.Data.Token; | ||
|
|
||
| webDirectory = await ScanAsync(httpClient, webDirectory); | ||
| } | ||
| } | ||
| else | ||
| { | ||
| webDirectory = await ScanAsync(httpClient, webDirectory); | ||
| } | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); | ||
| webDirectory.Error = true; | ||
|
|
||
| OpenDirectoryIndexer.Session.Errors++; | ||
|
|
||
| if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) | ||
| { | ||
| OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); | ||
| } | ||
|
|
||
| throw; | ||
| } | ||
|
|
||
| return webDirectory; | ||
| } | ||
|
|
||
| private static string GetDriveHash(WebDirectory webDirectory) | ||
| { | ||
| Match driveHashRegexMatch = FolderHashRegex.Match(webDirectory.Url); | ||
|
|
||
| if (!driveHashRegexMatch.Success) | ||
| { | ||
| throw new Exception("Error getting drivehash"); | ||
| } | ||
|
|
||
| return driveHashRegexMatch.Groups["FolderHash"].Value; | ||
| } | ||
|
|
||
| private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory) | ||
| { | ||
| Logger.Debug($"Retrieving listings for {webDirectory.Uri}"); | ||
|
|
||
| webDirectory.Parser = Parser; | ||
|
|
||
| try | ||
| { | ||
| string driveHash = GetDriveHash(webDirectory); | ||
|
|
||
| Logger.Warn($"Retrieving listings for {webDirectory.Uri}"); | ||
|
|
||
| HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetApiListingUrl(driveHash, OpenDirectoryIndexer.Session.Parameters[Constants.Parameters_GoFileIOAccountToken])); | ||
|
|
||
| webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode; | ||
| httpResponseMessage.EnsureSuccessStatusCode(); | ||
|
|
||
| string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); | ||
|
|
||
| GoFileIOListingResult indexResponse = GoFileIOListingResult.FromJson(responseJson); | ||
|
|
||
| if (indexResponse.Status != StatusOK) | ||
| { | ||
| throw new Exception($"Error retrieving listing for {webDirectory.Uri}. Error: {indexResponse.Status}"); | ||
| } | ||
|
|
||
| foreach (Content entry in indexResponse.Data.Contents.Values) | ||
| { | ||
| if (entry.Type == "folder") | ||
| { | ||
| webDirectory.Subdirectories.Add(new WebDirectory(webDirectory) | ||
| { | ||
| Parser = Parser, | ||
| Url = GetFolderUrl(entry.Id), | ||
| Name = entry.Name | ||
| }); | ||
| } | ||
| else | ||
| { | ||
| webDirectory.Files.Add(new WebFile | ||
| { | ||
| Url = entry.DirectLink.ToString(), | ||
| FileName = entry.Name, | ||
| FileSize = entry.Size | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); | ||
| webDirectory.Error = true; | ||
|
|
||
| OpenDirectoryIndexer.Session.Errors++; | ||
|
|
||
| if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) | ||
| { | ||
| OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); | ||
| } | ||
|
|
||
| //throw; | ||
| } | ||
|
|
||
| return webDirectory; | ||
| } | ||
|
|
||
| private static string GetFolderUrl(string driveHash) => $"https://gofile.io/d/{driveHash}"; | ||
| private static string GetApiListingUrl(string driveHash, string accountToken) => $"{ApiBaseAddress}/getContent?contentId={Uri.EscapeDataString(driveHash)}&token={Uri.EscapeDataString(accountToken)}&websiteToken=12345"; | ||
| } |