Skip to content

Commit

Permalink
- Add Mediafire.com indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
KoalaBear84 committed May 22, 2022
1 parent 3619464 commit 79e0bbb
Show file tree
Hide file tree
Showing 5 changed files with 358 additions and 0 deletions.
2 changes: 2 additions & 0 deletions src/OpenDirectoryDownloader/Constants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ public class Constants

public const string PixeldrainDomain = "pixeldrain.com";

public const string MediafireDomain = "www.mediafire.com";

public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss";
public const string Parameters_Password = "PASSWORD";
public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode";
Expand Down
6 changes: 6 additions & 0 deletions src/OpenDirectoryDownloader/DirectoryParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using OpenDirectoryDownloader.Site.GDIndex.Go2Index;
using OpenDirectoryDownloader.Site.GDIndex.GoIndex;
using OpenDirectoryDownloader.Site.GoFileIO;
using OpenDirectoryDownloader.Site.Mediafire;
using OpenDirectoryDownloader.Site.Pixeldrain;
using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -78,6 +79,11 @@ public static async Task<WebDirectory> ParseHtml(WebDirectory webDirectory, stri
return await PixeldrainParser.ParseIndex(httpClient, webDirectory, html);
}

if (webDirectory.Uri.Host == Constants.MediafireDomain)
{
return await MediafireParser.ParseIndex(httpClient, webDirectory);
}

if (httpClient is not null)
{
foreach (IHtmlScriptElement script in htmlDocument.Scripts.Where(s => s.Source is not null))
Expand Down
2 changes: 2 additions & 0 deletions src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,7 @@ public async void StartIndexingAsync()
Session.Root.Uri.Host != Constants.GoogleDriveDomain &&
Session.Root.Uri.Host != Constants.BlitzfilesTechDomain &&
Session.Root.Uri.Host != Constants.GoFileIoDomain &&
Session.Root.Uri.Host != Constants.MediafireDomain &&
Session.Root.Uri.Host != Constants.PixeldrainDomain)
{
if (Session.TotalFiles > 0)
Expand Down Expand Up @@ -450,6 +451,7 @@ public async void StartIndexingAsync()
Session.Root.Uri.Host != Constants.GoogleDriveDomain &&
Session.Root.Uri.Host != Constants.BlitzfilesTechDomain &&
Session.Root.Uri.Host != Constants.GoFileIoDomain &&
Session.Root.Uri.Host != Constants.MediafireDomain &&
Session.Root.Uri.Host != Constants.PixeldrainDomain)
{
if (Session.TotalFiles > 0)
Expand Down
134 changes: 134 additions & 0 deletions src/OpenDirectoryDownloader/Site/Mediafire/MediafireParser.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
using NLog;
using OpenDirectoryDownloader.Shared.Models;
using System;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace OpenDirectoryDownloader.Site.Mediafire;

public static class MediafireParser
{
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
private static readonly Regex FolderIdRegex = new Regex(@"\/folder\/(?<FolderId>[^/]*)(?:\/?.*)?");
private const string Parser = "Mediafire";
private const string StatusSuccess = "Success";
private const string ApiBaseAddress = "https://www.mediafire.com/api/1.4";

public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDirectory webDirectory)
{
try
{
webDirectory = await ScanAsync(httpClient, webDirectory);
}
catch (Exception ex)
{
Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}");
webDirectory.Error = true;

OpenDirectoryIndexer.Session.Errors++;

if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
{
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
}

throw;
}

return webDirectory;
}

private static string GetFolderId(WebDirectory webDirectory)
{
Match folderIdRegexMatch = FolderIdRegex.Match(webDirectory.Url);

if (!folderIdRegexMatch.Success)
{
throw new Exception("Error getting folder id");
}

return folderIdRegexMatch.Groups["FolderId"].Value;
}

private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory)
{
Logger.Debug($"Retrieving listings for {webDirectory.Uri}");

webDirectory.Parser = Parser;

try
{
string folderId = GetFolderId(webDirectory);

Logger.Warn($"Retrieving listings for {webDirectory.Uri}");

foreach (string listingType in new string[2] { "folders", "files" })
{
HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(GetApiListingUrl(folderId, listingType));

webDirectory.ParsedSuccessfully = httpResponseMessage.IsSuccessStatusCode;
httpResponseMessage.EnsureSuccessStatusCode();

string responseJson = await httpResponseMessage.Content.ReadAsStringAsync();

MediafireResult indexResponse = MediafireResult.FromJson(responseJson);

if (indexResponse.Response.Result != StatusSuccess)
{
throw new Exception($"Error retrieving listing for {webDirectory.Uri}. Error: {indexResponse.Response.Result}");
}

ProcessListing(webDirectory, indexResponse);
}
}
catch (Exception ex)
{
Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}");
webDirectory.Error = true;

OpenDirectoryIndexer.Session.Errors++;

if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url))
{
OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url);
}

//throw;
}

return webDirectory;
}

private static void ProcessListing(WebDirectory webDirectory, MediafireResult indexResponse)
{
if (indexResponse.Response.FolderContent.Folders is not null)
{
foreach (Folder folder in indexResponse.Response.FolderContent.Folders)
{
webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
{
Parser = Parser,
Url = GetFolderUrl(folder.Folderkey),
Name = folder.Name
});
}
}

if (indexResponse.Response.FolderContent.Files is not null)
{
foreach (File file in indexResponse.Response.FolderContent.Files)
{
webDirectory.Files.Add(new WebFile
{
Url = file.Links?.NormalDownload?.ToString(),
FileName = file.Filename,
FileSize = file.Size
});
}
}
}

private static string GetFolderUrl(string folderId) => $"https://www.mediafire.com/folder/{folderId}";
private static string GetApiListingUrl(string folderId, string type, int chunk = 1) => $"{ApiBaseAddress}/folder/get_content.php?content_type={type}&filter=all&order_by=name&order_direction=asc&chunk={chunk}&version=1.5&folder_key={folderId}&response_format=json";
}
214 changes: 214 additions & 0 deletions src/OpenDirectoryDownloader/Site/Mediafire/MediafireResult.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
// <auto-generated />
//
// To parse this JSON data, add NuGet 'Newtonsoft.Json' then do:
//
// using QuickType;
//
// var mediafireResult = MediafireResult.FromJson(jsonString);

namespace OpenDirectoryDownloader.Site.Mediafire
{
using System;
using System.Collections.Generic;

using System.Globalization;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;

public partial class MediafireResult
{
[JsonProperty("response")]
public Response Response { get; set; }
}

public partial class Response
{
[JsonProperty("action")]
public string Action { get; set; }

[JsonProperty("asynchronous")]
public string Asynchronous { get; set; }

[JsonProperty("folder_content")]
public FolderContent FolderContent { get; set; }

[JsonProperty("result")]
public string Result { get; set; }

[JsonProperty("current_api_version")]
public string CurrentApiVersion { get; set; }
}

public partial class FolderContent
{
[JsonProperty("chunk_size")]
public long ChunkSize { get; set; }

[JsonProperty("content_type")]
public string ContentType { get; set; }

[JsonProperty("chunk_number")]
public long ChunkNumber { get; set; }

[JsonProperty("folderkey")]
public string Folderkey { get; set; }

[JsonProperty("folders")]
public Folder[] Folders { get; set; }

[JsonProperty("files")]
public File[] Files { get; set; }

[JsonProperty("more_chunks")]
public string MoreChunks { get; set; }

[JsonProperty("revision")]
public long Revision { get; set; }
}

public partial class File
{
[JsonProperty("quickkey")]
public string Quickkey { get; set; }

[JsonProperty("hash")]
public string Hash { get; set; }

[JsonProperty("filename")]
public string Filename { get; set; }

[JsonProperty("description")]
public string Description { get; set; }

[JsonProperty("size")]
public long Size { get; set; }

[JsonProperty("privacy")]
public string Privacy { get; set; }

[JsonProperty("created")]
public DateTimeOffset Created { get; set; }

[JsonProperty("password_protected")]
public string PasswordProtected { get; set; }

[JsonProperty("mimetype")]
public string Mimetype { get; set; }

[JsonProperty("filetype")]
public string Filetype { get; set; }

[JsonProperty("view")]
public long View { get; set; }

[JsonProperty("edit")]
public long Edit { get; set; }

[JsonProperty("revision")]
public long Revision { get; set; }

[JsonProperty("flag")]
public long Flag { get; set; }

[JsonProperty("permissions")]
public Permissions Permissions { get; set; }

[JsonProperty("downloads")]
public long Downloads { get; set; }

[JsonProperty("views")]
public long Views { get; set; }

[JsonProperty("links")]
public Links Links { get; set; }

[JsonProperty("created_utc")]
public DateTimeOffset CreatedUtc { get; set; }
}

public partial class Links
{
[JsonProperty("normal_download")]
public Uri NormalDownload { get; set; }
}

public partial class Permissions
{
[JsonProperty("value")]
public long Value { get; set; }

[JsonProperty("explicit")]
public long Explicit { get; set; }

[JsonProperty("read")]
public long Read { get; set; }

[JsonProperty("write")]
public long Write { get; set; }
}

public partial class Folder
{
[JsonProperty("folderkey")]
public string Folderkey { get; set; }

[JsonProperty("name")]
public string Name { get; set; }

[JsonProperty("description")]
public string Description { get; set; }

[JsonProperty("tags")]
public string Tags { get; set; }

[JsonProperty("privacy")]
public string Privacy { get; set; }

[JsonProperty("created")]
public DateTimeOffset Created { get; set; }

[JsonProperty("revision")]
public long Revision { get; set; }

[JsonProperty("flag")]
public long Flag { get; set; }

[JsonProperty("permissions")]
public Permissions Permissions { get; set; }

[JsonProperty("file_count")]
public long FileCount { get; set; }

[JsonProperty("folder_count")]
public long FolderCount { get; set; }

[JsonProperty("dropbox_enabled")]
public string DropboxEnabled { get; set; }

[JsonProperty("created_utc")]
public DateTimeOffset CreatedUtc { get; set; }
}

public partial class MediafireResult
{
public static MediafireResult FromJson(string json) => JsonConvert.DeserializeObject<MediafireResult>(json, Converter.Settings);
}

public static class Serialize
{
public static string ToJson(this MediafireResult self) => JsonConvert.SerializeObject(self, Converter.Settings);
}

internal static class Converter
{
public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings
{
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
DateParseHandling = DateParseHandling.None,
Converters =
{
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
},
};
}
}

0 comments on commit 79e0bbb

Please sign in to comment.