diff --git a/src/OpenDirectoryDownloader/Constants.cs b/src/OpenDirectoryDownloader/Constants.cs index 0c7097f1..8ad28420 100644 --- a/src/OpenDirectoryDownloader/Constants.cs +++ b/src/OpenDirectoryDownloader/Constants.cs @@ -11,6 +11,8 @@ public class Constants public const string PixeldrainDomain = "pixeldrain.com"; + public const string MediafireDomain = "www.mediafire.com"; + public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss"; public const string Parameters_Password = "PASSWORD"; public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode"; diff --git a/src/OpenDirectoryDownloader/DirectoryParser.cs b/src/OpenDirectoryDownloader/DirectoryParser.cs index 968aee61..3123138b 100644 --- a/src/OpenDirectoryDownloader/DirectoryParser.cs +++ b/src/OpenDirectoryDownloader/DirectoryParser.cs @@ -14,6 +14,7 @@ using OpenDirectoryDownloader.Site.GDIndex.Go2Index; using OpenDirectoryDownloader.Site.GDIndex.GoIndex; using OpenDirectoryDownloader.Site.GoFileIO; +using OpenDirectoryDownloader.Site.Mediafire; using OpenDirectoryDownloader.Site.Pixeldrain; using System; using System.Collections.Generic; @@ -78,6 +79,11 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri return await PixeldrainParser.ParseIndex(httpClient, webDirectory, html); } + if (webDirectory.Uri.Host == Constants.MediafireDomain) + { + return await MediafireParser.ParseIndex(httpClient, webDirectory); + } + if (httpClient is not null) { foreach (IHtmlScriptElement script in htmlDocument.Scripts.Where(s => s.Source is not null)) diff --git a/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs b/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs index 74749364..aaa51f53 100644 --- a/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs +++ b/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs @@ -380,6 +380,7 @@ public async void StartIndexingAsync() Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain && Session.Root.Uri.Host != Constants.GoFileIoDomain && + Session.Root.Uri.Host != Constants.MediafireDomain && Session.Root.Uri.Host != Constants.PixeldrainDomain) { if (Session.TotalFiles > 0) @@ -450,6 +451,7 @@ public async void StartIndexingAsync() Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain && Session.Root.Uri.Host != Constants.GoFileIoDomain && + Session.Root.Uri.Host != Constants.MediafireDomain && Session.Root.Uri.Host != Constants.PixeldrainDomain) { if (Session.TotalFiles > 0) diff --git a/src/OpenDirectoryDownloader/Site/Mediafire/MediafireParser.cs b/src/OpenDirectoryDownloader/Site/Mediafire/MediafireParser.cs new file mode 100644 index 00000000..64d5e24f --- /dev/null +++ b/src/OpenDirectoryDownloader/Site/Mediafire/MediafireParser.cs @@ -0,0 +1,134 @@ +using NLog; +using OpenDirectoryDownloader.Shared.Models; +using System; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace OpenDirectoryDownloader.Site.Mediafire; + +public static class MediafireParser +{ + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + private static readonly Regex FolderIdRegex = new Regex(@"\/folder\/(?[^/]*)(?:\/?.*)?"); + private const string Parser = "Mediafire"; + private const string StatusSuccess = "Success"; + private const string ApiBaseAddress = "https://www.mediafire.com/api/1.4"; + + public static async Task ParseIndex(HttpClient httpClient, WebDirectory webDirectory) + { + try + { + webDirectory = await ScanAsync(httpClient, webDirectory); + } + catch (Exception ex) + { + Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); + webDirectory.Error = true; + + OpenDirectoryIndexer.Session.Errors++; + + if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) + { + OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); + } + + throw; + } + + return webDirectory; + } + + private static string GetFolderId(WebDirectory webDirectory) + { + Match folderIdRegexMatch = FolderIdRegex.Match(webDirectory.Url); + + if (!folderIdRegexMatch.Success) + { + throw new Exception("Error getting folder id"); + } + + return folderIdRegexMatch.Groups["FolderId"].Value; + } + + private static async Task ScanAsync(HttpClient httpClient, WebDirectory webDirectory) + { + Logger.Debug($"Retrieving listings for {webDirectory.Uri}"); + + webDirectory.Parser = Parser; + + try + { + string folderId = GetFolderId(webDirectory); + + Logger.Warn($"Retrieving listings for {webDirectory.Uri}"); + + foreach (string listingType in new string[2] { "folders", "files" }) + { + HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetApiListingUrl(folderId, listingType)); + + webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode; + httpResponseMessage.EnsureSuccessStatusCode(); + + string responseJson = await httpResponseMessage.Content.ReadAsStringAsync(); + + MediafireResult indexResponse = MediafireResult.FromJson(responseJson); + + if (indexResponse.Response.Result != StatusSuccess) + { + throw new Exception($"Error retrieving listing for {webDirectory.Uri}. Error: {indexResponse.Response.Result}"); + } + + ProcessListing(webDirectory, indexResponse); + } + } + catch (Exception ex) + { + Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); + webDirectory.Error = true; + + OpenDirectoryIndexer.Session.Errors++; + + if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) + { + OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); + } + + //throw; + } + + return webDirectory; + } + + private static void ProcessListing(WebDirectory webDirectory, MediafireResult indexResponse) + { + if (indexResponse.Response.FolderContent.Folders is not null) + { + foreach (Folder folder in indexResponse.Response.FolderContent.Folders) + { + webDirectory.Subdirectories.Add(new WebDirectory(webDirectory) + { + Parser = Parser, + Url = GetFolderUrl(folder.Folderkey), + Name = folder.Name + }); + } + } + + if (indexResponse.Response.FolderContent.Files is not null) + { + foreach (File file in indexResponse.Response.FolderContent.Files) + { + webDirectory.Files.Add(new WebFile + { + Url = file.Links?.NormalDownload?.ToString(), + FileName = file.Filename, + FileSize = file.Size + }); + } + } + } + + private static string GetFolderUrl(string folderId) => $"https://www.mediafire.com/folder/{folderId}"; + private static string GetApiListingUrl(string folderId, string type, int chunk = 1) => $"{ApiBaseAddress}/folder/get_content.php?content_type={type}&filter=all&order_by=name&order_direction=asc&chunk={chunk}&version=1.5&folder_key={folderId}&response_format=json"; +} diff --git a/src/OpenDirectoryDownloader/Site/Mediafire/MediafireResult.cs b/src/OpenDirectoryDownloader/Site/Mediafire/MediafireResult.cs new file mode 100644 index 00000000..6d53bc09 --- /dev/null +++ b/src/OpenDirectoryDownloader/Site/Mediafire/MediafireResult.cs @@ -0,0 +1,214 @@ +// +// +// To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: +// +// using QuickType; +// +// var mediafireResult = MediafireResult.FromJson(jsonString); + +namespace OpenDirectoryDownloader.Site.Mediafire +{ + using System; + using System.Collections.Generic; + + using System.Globalization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + public partial class MediafireResult + { + [JsonProperty("response")] + public Response Response { get; set; } + } + + public partial class Response + { + [JsonProperty("action")] + public string Action { get; set; } + + [JsonProperty("asynchronous")] + public string Asynchronous { get; set; } + + [JsonProperty("folder_content")] + public FolderContent FolderContent { get; set; } + + [JsonProperty("result")] + public string Result { get; set; } + + [JsonProperty("current_api_version")] + public string CurrentApiVersion { get; set; } + } + + public partial class FolderContent + { + [JsonProperty("chunk_size")] + public long ChunkSize { get; set; } + + [JsonProperty("content_type")] + public string ContentType { get; set; } + + [JsonProperty("chunk_number")] + public long ChunkNumber { get; set; } + + [JsonProperty("folderkey")] + public string Folderkey { get; set; } + + [JsonProperty("folders")] + public Folder[] Folders { get; set; } + + [JsonProperty("files")] + public File[] Files { get; set; } + + [JsonProperty("more_chunks")] + public string MoreChunks { get; set; } + + [JsonProperty("revision")] + public long Revision { get; set; } + } + + public partial class File + { + [JsonProperty("quickkey")] + public string Quickkey { get; set; } + + [JsonProperty("hash")] + public string Hash { get; set; } + + [JsonProperty("filename")] + public string Filename { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("size")] + public long Size { get; set; } + + [JsonProperty("privacy")] + public string Privacy { get; set; } + + [JsonProperty("created")] + public DateTimeOffset Created { get; set; } + + [JsonProperty("password_protected")] + public string PasswordProtected { get; set; } + + [JsonProperty("mimetype")] + public string Mimetype { get; set; } + + [JsonProperty("filetype")] + public string Filetype { get; set; } + + [JsonProperty("view")] + public long View { get; set; } + + [JsonProperty("edit")] + public long Edit { get; set; } + + [JsonProperty("revision")] + public long Revision { get; set; } + + [JsonProperty("flag")] + public long Flag { get; set; } + + [JsonProperty("permissions")] + public Permissions Permissions { get; set; } + + [JsonProperty("downloads")] + public long Downloads { get; set; } + + [JsonProperty("views")] + public long Views { get; set; } + + [JsonProperty("links")] + public Links Links { get; set; } + + [JsonProperty("created_utc")] + public DateTimeOffset CreatedUtc { get; set; } + } + + public partial class Links + { + [JsonProperty("normal_download")] + public Uri NormalDownload { get; set; } + } + + public partial class Permissions + { + [JsonProperty("value")] + public long Value { get; set; } + + [JsonProperty("explicit")] + public long Explicit { get; set; } + + [JsonProperty("read")] + public long Read { get; set; } + + [JsonProperty("write")] + public long Write { get; set; } + } + + public partial class Folder + { + [JsonProperty("folderkey")] + public string Folderkey { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("tags")] + public string Tags { get; set; } + + [JsonProperty("privacy")] + public string Privacy { get; set; } + + [JsonProperty("created")] + public DateTimeOffset Created { get; set; } + + [JsonProperty("revision")] + public long Revision { get; set; } + + [JsonProperty("flag")] + public long Flag { get; set; } + + [JsonProperty("permissions")] + public Permissions Permissions { get; set; } + + [JsonProperty("file_count")] + public long FileCount { get; set; } + + [JsonProperty("folder_count")] + public long FolderCount { get; set; } + + [JsonProperty("dropbox_enabled")] + public string DropboxEnabled { get; set; } + + [JsonProperty("created_utc")] + public DateTimeOffset CreatedUtc { get; set; } + } + + public partial class MediafireResult + { + public static MediafireResult FromJson(string json) => JsonConvert.DeserializeObject(json, Converter.Settings); + } + + public static class Serialize + { + public static string ToJson(this MediafireResult self) => JsonConvert.SerializeObject(self, Converter.Settings); + } + + internal static class Converter + { + public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + DateParseHandling = DateParseHandling.None, + Converters = + { + new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } + }, + }; + } +}