-
-
Notifications
You must be signed in to change notification settings - Fork 95
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
3619464
commit 79e0bbb
Showing
5 changed files
with
358 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
134 changes: 134 additions & 0 deletions
134
src/OpenDirectoryDownloader/Site/Mediafire/MediafireParser.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,134 @@ | ||
using NLog; | ||
using OpenDirectoryDownloader.Shared.Models; | ||
using System; | ||
using System.Net.Http; | ||
using System.Text.RegularExpressions; | ||
using System.Threading.Tasks; | ||
|
||
namespace OpenDirectoryDownloader.Site.Mediafire; | ||
|
||
public static class MediafireParser | ||
{ | ||
private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); | ||
private static readonly Regex FolderIdRegex = new Regex(@"\/folder\/(?<FolderId>[^/]*)(?:\/?.*)?"); | ||
private const string Parser = "Mediafire"; | ||
private const string StatusSuccess = "Success"; | ||
private const string ApiBaseAddress = "https://www.mediafire.com/api/1.4"; | ||
|
||
public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory) | ||
{ | ||
try | ||
{ | ||
webDirectory = await ScanAsync(httpClient, webDirectory); | ||
} | ||
catch (Exception ex) | ||
{ | ||
Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); | ||
webDirectory.Error = true; | ||
|
||
OpenDirectoryIndexer.Session.Errors++; | ||
|
||
if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) | ||
{ | ||
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); | ||
} | ||
|
||
throw; | ||
} | ||
|
||
return webDirectory; | ||
} | ||
|
||
private static string GetFolderId(WebDirectory webDirectory) | ||
{ | ||
Match folderIdRegexMatch = FolderIdRegex.Match(webDirectory.Url); | ||
|
||
if (!folderIdRegexMatch.Success) | ||
{ | ||
throw new Exception("Error getting folder id"); | ||
} | ||
|
||
return folderIdRegexMatch.Groups["FolderId"].Value; | ||
} | ||
|
||
private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory) | ||
{ | ||
Logger.Debug($"Retrieving listings for {webDirectory.Uri}"); | ||
|
||
webDirectory.Parser = Parser; | ||
|
||
try | ||
{ | ||
string folderId = GetFolderId(webDirectory); | ||
|
||
Logger.Warn($"Retrieving listings for {webDirectory.Uri}"); | ||
|
||
foreach (string listingType in new string[2] { "folders", "files" }) | ||
{ | ||
HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetApiListingUrl(folderId, listingType)); | ||
|
||
webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode; | ||
httpResponseMessage.EnsureSuccessStatusCode(); | ||
|
||
string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); | ||
|
||
MediafireResult indexResponse = MediafireResult.FromJson(responseJson); | ||
|
||
if (indexResponse.Response.Result != StatusSuccess) | ||
{ | ||
throw new Exception($"Error retrieving listing for {webDirectory.Uri}. Error: {indexResponse.Response.Result}"); | ||
} | ||
|
||
ProcessListing(webDirectory, indexResponse); | ||
} | ||
} | ||
catch (Exception ex) | ||
{ | ||
Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); | ||
webDirectory.Error = true; | ||
|
||
OpenDirectoryIndexer.Session.Errors++; | ||
|
||
if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) | ||
{ | ||
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); | ||
} | ||
|
||
//throw; | ||
} | ||
|
||
return webDirectory; | ||
} | ||
|
||
private static void ProcessListing(WebDirectory webDirectory, MediafireResult indexResponse) | ||
{ | ||
if (indexResponse.Response.FolderContent.Folders is not null) | ||
{ | ||
foreach (Folder folder in indexResponse.Response.FolderContent.Folders) | ||
{ | ||
webDirectory.Subdirectories.Add(new WebDirectory(webDirectory) | ||
{ | ||
Parser = Parser, | ||
Url = GetFolderUrl(folder.Folderkey), | ||
Name = folder.Name | ||
}); | ||
} | ||
} | ||
|
||
if (indexResponse.Response.FolderContent.Files is not null) | ||
{ | ||
foreach (File file in indexResponse.Response.FolderContent.Files) | ||
{ | ||
webDirectory.Files.Add(new WebFile | ||
{ | ||
Url = file.Links?.NormalDownload?.ToString(), | ||
FileName = file.Filename, | ||
FileSize = file.Size | ||
}); | ||
} | ||
} | ||
} | ||
|
||
private static string GetFolderUrl(string folderId) => $"https://www.mediafire.com/folder/{folderId}"; | ||
private static string GetApiListingUrl(string folderId, string type, int chunk = 1) => $"{ApiBaseAddress}/folder/get_content.php?content_type={type}&filter=all&order_by=name&order_direction=asc&chunk={chunk}&version=1.5&folder_key={folderId}&response_format=json"; | ||
} |
214 changes: 214 additions & 0 deletions
214
src/OpenDirectoryDownloader/Site/Mediafire/MediafireResult.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,214 @@ | ||
// <auto-generated /> | ||
// | ||
// To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: | ||
// | ||
// using QuickType; | ||
// | ||
// var mediafireResult = MediafireResult.FromJson(jsonString); | ||
|
||
namespace OpenDirectoryDownloader.Site.Mediafire | ||
{ | ||
using System; | ||
using System.Collections.Generic; | ||
|
||
using System.Globalization; | ||
using Newtonsoft.Json; | ||
using Newtonsoft.Json.Converters; | ||
|
||
public partial class MediafireResult | ||
{ | ||
[JsonProperty("response")] | ||
public Response Response { get; set; } | ||
} | ||
|
||
public partial class Response | ||
{ | ||
[JsonProperty("action")] | ||
public string Action { get; set; } | ||
|
||
[JsonProperty("asynchronous")] | ||
public string Asynchronous { get; set; } | ||
|
||
[JsonProperty("folder_content")] | ||
public FolderContent FolderContent { get; set; } | ||
|
||
[JsonProperty("result")] | ||
public string Result { get; set; } | ||
|
||
[JsonProperty("current_api_version")] | ||
public string CurrentApiVersion { get; set; } | ||
} | ||
|
||
public partial class FolderContent | ||
{ | ||
[JsonProperty("chunk_size")] | ||
public long ChunkSize { get; set; } | ||
|
||
[JsonProperty("content_type")] | ||
public string ContentType { get; set; } | ||
|
||
[JsonProperty("chunk_number")] | ||
public long ChunkNumber { get; set; } | ||
|
||
[JsonProperty("folderkey")] | ||
public string Folderkey { get; set; } | ||
|
||
[JsonProperty("folders")] | ||
public Folder[] Folders { get; set; } | ||
|
||
[JsonProperty("files")] | ||
public File[] Files { get; set; } | ||
|
||
[JsonProperty("more_chunks")] | ||
public string MoreChunks { get; set; } | ||
|
||
[JsonProperty("revision")] | ||
public long Revision { get; set; } | ||
} | ||
|
||
public partial class File | ||
{ | ||
[JsonProperty("quickkey")] | ||
public string Quickkey { get; set; } | ||
|
||
[JsonProperty("hash")] | ||
public string Hash { get; set; } | ||
|
||
[JsonProperty("filename")] | ||
public string Filename { get; set; } | ||
|
||
[JsonProperty("description")] | ||
public string Description { get; set; } | ||
|
||
[JsonProperty("size")] | ||
public long Size { get; set; } | ||
|
||
[JsonProperty("privacy")] | ||
public string Privacy { get; set; } | ||
|
||
[JsonProperty("created")] | ||
public DateTimeOffset Created { get; set; } | ||
|
||
[JsonProperty("password_protected")] | ||
public string PasswordProtected { get; set; } | ||
|
||
[JsonProperty("mimetype")] | ||
public string Mimetype { get; set; } | ||
|
||
[JsonProperty("filetype")] | ||
public string Filetype { get; set; } | ||
|
||
[JsonProperty("view")] | ||
public long View { get; set; } | ||
|
||
[JsonProperty("edit")] | ||
public long Edit { get; set; } | ||
|
||
[JsonProperty("revision")] | ||
public long Revision { get; set; } | ||
|
||
[JsonProperty("flag")] | ||
public long Flag { get; set; } | ||
|
||
[JsonProperty("permissions")] | ||
public Permissions Permissions { get; set; } | ||
|
||
[JsonProperty("downloads")] | ||
public long Downloads { get; set; } | ||
|
||
[JsonProperty("views")] | ||
public long Views { get; set; } | ||
|
||
[JsonProperty("links")] | ||
public Links Links { get; set; } | ||
|
||
[JsonProperty("created_utc")] | ||
public DateTimeOffset CreatedUtc { get; set; } | ||
} | ||
|
||
public partial class Links | ||
{ | ||
[JsonProperty("normal_download")] | ||
public Uri NormalDownload { get; set; } | ||
} | ||
|
||
public partial class Permissions | ||
{ | ||
[JsonProperty("value")] | ||
public long Value { get; set; } | ||
|
||
[JsonProperty("explicit")] | ||
public long Explicit { get; set; } | ||
|
||
[JsonProperty("read")] | ||
public long Read { get; set; } | ||
|
||
[JsonProperty("write")] | ||
public long Write { get; set; } | ||
} | ||
|
||
public partial class Folder | ||
{ | ||
[JsonProperty("folderkey")] | ||
public string Folderkey { get; set; } | ||
|
||
[JsonProperty("name")] | ||
public string Name { get; set; } | ||
|
||
[JsonProperty("description")] | ||
public string Description { get; set; } | ||
|
||
[JsonProperty("tags")] | ||
public string Tags { get; set; } | ||
|
||
[JsonProperty("privacy")] | ||
public string Privacy { get; set; } | ||
|
||
[JsonProperty("created")] | ||
public DateTimeOffset Created { get; set; } | ||
|
||
[JsonProperty("revision")] | ||
public long Revision { get; set; } | ||
|
||
[JsonProperty("flag")] | ||
public long Flag { get; set; } | ||
|
||
[JsonProperty("permissions")] | ||
public Permissions Permissions { get; set; } | ||
|
||
[JsonProperty("file_count")] | ||
public long FileCount { get; set; } | ||
|
||
[JsonProperty("folder_count")] | ||
public long FolderCount { get; set; } | ||
|
||
[JsonProperty("dropbox_enabled")] | ||
public string DropboxEnabled { get; set; } | ||
|
||
[JsonProperty("created_utc")] | ||
public DateTimeOffset CreatedUtc { get; set; } | ||
} | ||
|
||
public partial class MediafireResult | ||
{ | ||
public static MediafireResult FromJson(string json) => JsonConvert.DeserializeObject<MediafireResult>(json, Converter.Settings); | ||
} | ||
|
||
public static class Serialize | ||
{ | ||
public static string ToJson(this MediafireResult self) => JsonConvert.SerializeObject(self, Converter.Settings); | ||
} | ||
|
||
internal static class Converter | ||
{ | ||
public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings | ||
{ | ||
MetadataPropertyHandling = MetadataPropertyHandling.Ignore, | ||
DateParseHandling = DateParseHandling.None, | ||
Converters = | ||
{ | ||
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } | ||
}, | ||
}; | ||
} | ||
} |