From cef6286fdf542398db11419baf7d7850d3abfaba Mon Sep 17 00:00:00 2001 From: KoalaBear Date: Sun, 22 May 2022 13:25:00 +0200 Subject: [PATCH] - Add Pixeldrain.com indexing --- src/OpenDirectoryDownloader/Constants.cs | 2 + .../DirectoryParser.cs | 6 + .../OpenDirectoryIndexer.cs | 6 +- .../Site/GoFileIO/GoFileIOParser.cs | 3 - .../Site/Pixeldrain/PixeldrainFileResult.cs | 124 ++++++++++++++ .../Site/Pixeldrain/PixeldrainListResult.cs | 151 ++++++++++++++++++ .../Site/Pixeldrain/PixeldrainParser.cs | 122 ++++++++++++++ 7 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainFileResult.cs create mode 100644 src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainListResult.cs create mode 100644 src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainParser.cs diff --git a/src/OpenDirectoryDownloader/Constants.cs b/src/OpenDirectoryDownloader/Constants.cs index 8fd13497..0c7097f1 100644 --- a/src/OpenDirectoryDownloader/Constants.cs +++ b/src/OpenDirectoryDownloader/Constants.cs @@ -9,6 +9,8 @@ public class Constants public const string Parameters_GdIndex_RootId = "GDINDEX_ROOTID"; public const string Parameters_GoFileIOAccountToken = "GOFILE_ACCOUNTTOKEN"; + public const string PixeldrainDomain = "pixeldrain.com"; + public const string DateTimeFormat = "yyyy-MM-dd HH:mm:ss"; public const string Parameters_Password = "PASSWORD"; public const string Parameters_FtpEncryptionMode = "FtpEncryptionMode"; diff --git a/src/OpenDirectoryDownloader/DirectoryParser.cs b/src/OpenDirectoryDownloader/DirectoryParser.cs index 7bfba31e..968aee61 100644 --- a/src/OpenDirectoryDownloader/DirectoryParser.cs +++ b/src/OpenDirectoryDownloader/DirectoryParser.cs @@ -14,6 +14,7 @@ using OpenDirectoryDownloader.Site.GDIndex.Go2Index; using OpenDirectoryDownloader.Site.GDIndex.GoIndex; using OpenDirectoryDownloader.Site.GoFileIO; +using OpenDirectoryDownloader.Site.Pixeldrain; using System; using System.Collections.Generic; using System.Diagnostics; @@ -72,6 +73,11 @@ public static async Task ParseHtml(WebDirectory webDirectory, stri return await GoFileIOParser.ParseIndex(httpClient, webDirectory); } + if (webDirectory.Uri.Host == Constants.PixeldrainDomain) + { + return await PixeldrainParser.ParseIndex(httpClient, webDirectory, html); + } + if (httpClient is not null) { foreach (IHtmlScriptElement script in htmlDocument.Scripts.Where(s => s.Source is not null)) diff --git a/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs b/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs index 49fba0bc..74749364 100644 --- a/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs +++ b/src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs @@ -379,7 +379,8 @@ public async void StartIndexingAsync() if (!OpenDirectoryIndexerSettings.CommandLineOptions.NoUrls && Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain && - Session.Root.Uri.Host != Constants.GoFileIoDomain) + Session.Root.Uri.Host != Constants.GoFileIoDomain && + Session.Root.Uri.Host != Constants.PixeldrainDomain) { if (Session.TotalFiles > 0) { @@ -448,7 +449,8 @@ public async void StartIndexingAsync() if (OpenDirectoryIndexerSettings.CommandLineOptions.Speedtest && Session.Root.Uri.Host != Constants.GoogleDriveDomain && Session.Root.Uri.Host != Constants.BlitzfilesTechDomain && - Session.Root.Uri.Host != Constants.GoFileIoDomain) + Session.Root.Uri.Host != Constants.GoFileIoDomain && + Session.Root.Uri.Host != Constants.PixeldrainDomain) { if (Session.TotalFiles > 0) { diff --git a/src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOParser.cs b/src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOParser.cs index a7679e01..79c9b553 100644 --- a/src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOParser.cs +++ b/src/OpenDirectoryDownloader/Site/GoFileIO/GoFileIOParser.cs @@ -7,9 +7,6 @@ namespace OpenDirectoryDownloader.Site.GoFileIO; -/// -/// Similar to GoFile.IO -/// public static class GoFileIOParser { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); diff --git a/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainFileResult.cs b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainFileResult.cs new file mode 100644 index 00000000..3f5acb20 --- /dev/null +++ b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainFileResult.cs @@ -0,0 +1,124 @@ +// +// +// To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: +// +// using QuickType; +// +// var pixeldrainFileResult = PixeldrainFileResult.FromJson(jsonString); + +namespace OpenDirectoryDownloader.Site.Pixeldrain.FileResult +{ + using System; + using System.Collections.Generic; + + using System.Globalization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + public partial class PixeldrainFileResult + { + [JsonProperty("type")] + public string Type { get; set; } + + [JsonProperty("api_response")] + public ApiResponse ApiResponse { get; set; } + + [JsonProperty("captcha_key")] + public string CaptchaKey { get; set; } + + [JsonProperty("view_token")] + public string ViewToken { get; set; } + + [JsonProperty("embedded")] + public bool Embedded { get; set; } + + [JsonProperty("user_ads_enabled")] + public bool UserAdsEnabled { get; set; } + } + + public partial class ApiResponse + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("size")] + public long Size { get; set; } + + [JsonProperty("views")] + public long Views { get; set; } + + [JsonProperty("bandwidth_used")] + public long BandwidthUsed { get; set; } + + [JsonProperty("bandwidth_used_paid")] + public long BandwidthUsedPaid { get; set; } + + [JsonProperty("downloads")] + public long Downloads { get; set; } + + [JsonProperty("date_upload")] + public DateTimeOffset DateUpload { get; set; } + + [JsonProperty("date_last_view")] + public DateTimeOffset DateLastView { get; set; } + + [JsonProperty("mime_type")] + public string MimeType { get; set; } + + [JsonProperty("thumbnail_href")] + public string ThumbnailHref { get; set; } + + [JsonProperty("hash_sha256")] + public string HashSha256 { get; set; } + + [JsonProperty("availability")] + public string Availability { get; set; } + + [JsonProperty("availability_message")] + public string AvailabilityMessage { get; set; } + + [JsonProperty("abuse_type")] + public string AbuseType { get; set; } + + [JsonProperty("abuse_reporter_name")] + public string AbuseReporterName { get; set; } + + [JsonProperty("can_edit")] + public bool CanEdit { get; set; } + + [JsonProperty("show_ads")] + public bool ShowAds { get; set; } + + [JsonProperty("allow_video_player")] + public bool AllowVideoPlayer { get; set; } + + [JsonProperty("download_speed_limit")] + public long DownloadSpeedLimit { get; set; } + } + + public partial class PixeldrainFileResult + { + public static PixeldrainFileResult FromJson(string json) => JsonConvert.DeserializeObject(json, Converter.Settings); + } + + public static class Serialize + { + public static string ToJson(this PixeldrainFileResult self) => JsonConvert.SerializeObject(self, Converter.Settings); + } + + internal static class Converter + { + public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + DateParseHandling = DateParseHandling.None, + Converters = + { + new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } + }, + }; + } +} diff --git a/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainListResult.cs b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainListResult.cs new file mode 100644 index 00000000..4548d641 --- /dev/null +++ b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainListResult.cs @@ -0,0 +1,151 @@ +// +// +// To parse this JSON data, add NuGet 'Newtonsoft.Json' then do: +// +// using QuickType; +// +// var pixeldrainListResult = PixeldrainListResult.FromJson(jsonString); + +namespace OpenDirectoryDownloader.Site.Pixeldrain.ListResult +{ + using System; + using System.Collections.Generic; + + using System.Globalization; + using Newtonsoft.Json; + using Newtonsoft.Json.Converters; + + public partial class PixeldrainListResult + { + [JsonProperty("type")] + public string Type { get; set; } + + [JsonProperty("api_response")] + public ApiResponse ApiResponse { get; set; } + + [JsonProperty("captcha_key")] + public string CaptchaKey { get; set; } + + [JsonProperty("view_token")] + public string ViewToken { get; set; } + + [JsonProperty("embedded")] + public bool Embedded { get; set; } + + [JsonProperty("user_ads_enabled")] + public bool UserAdsEnabled { get; set; } + } + + public partial class ApiResponse + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("date_created")] + public DateTimeOffset DateCreated { get; set; } + + [JsonProperty("file_count")] + public long FileCount { get; set; } + + [JsonProperty("files")] + public File[] Files { get; set; } + + [JsonProperty("can_edit")] + public bool CanEdit { get; set; } + } + + public partial class File + { + [JsonProperty("detail_href")] + public string DetailHref { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("size")] + public long Size { get; set; } + + [JsonProperty("views")] + public long Views { get; set; } + + [JsonProperty("bandwidth_used")] + public long BandwidthUsed { get; set; } + + [JsonProperty("bandwidth_used_paid")] + public long BandwidthUsedPaid { get; set; } + + [JsonProperty("downloads")] + public long Downloads { get; set; } + + [JsonProperty("date_upload")] + public DateTimeOffset DateUpload { get; set; } + + [JsonProperty("date_last_view")] + public DateTimeOffset DateLastView { get; set; } + + [JsonProperty("mime_type")] + public string MimeType { get; set; } + + [JsonProperty("thumbnail_href")] + public string ThumbnailHref { get; set; } + + [JsonProperty("hash_sha256")] + public string HashSha256 { get; set; } + + [JsonProperty("availability")] + public string Availability { get; set; } + + [JsonProperty("availability_message")] + public string AvailabilityMessage { get; set; } + + [JsonProperty("abuse_type")] + public string AbuseType { get; set; } + + [JsonProperty("abuse_reporter_name")] + public string AbuseReporterName { get; set; } + + [JsonProperty("can_edit")] + public bool CanEdit { get; set; } + + [JsonProperty("show_ads")] + public bool ShowAds { get; set; } + + [JsonProperty("allow_video_player")] + public bool AllowVideoPlayer { get; set; } + + [JsonProperty("download_speed_limit")] + public long DownloadSpeedLimit { get; set; } + } + + public partial class PixeldrainListResult + { + public static PixeldrainListResult FromJson(string json) => JsonConvert.DeserializeObject(json, Converter.Settings); + } + + public static class Serialize + { + public static string ToJson(this PixeldrainListResult self) => JsonConvert.SerializeObject(self, Converter.Settings); + } + + internal static class Converter + { + public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + DateParseHandling = DateParseHandling.None, + Converters = + { + new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } + }, + }; + } +} diff --git a/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainParser.cs b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainParser.cs new file mode 100644 index 00000000..3f4f27e0 --- /dev/null +++ b/src/OpenDirectoryDownloader/Site/Pixeldrain/PixeldrainParser.cs @@ -0,0 +1,122 @@ +using NLog; +using OpenDirectoryDownloader.Shared.Models; +using OpenDirectoryDownloader.Site.Pixeldrain.FileResult; +using OpenDirectoryDownloader.Site.Pixeldrain.ListResult; +using System; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace OpenDirectoryDownloader.Site.Pixeldrain; + +public static class PixeldrainParser +{ + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + private static readonly Regex ListingTypeRegex = new Regex(@".*\/(?.*)\/.*"); + private static readonly Regex ListingRegex = new Regex(@"window\.viewer_data = (?.*);"); + private const string Parser = "Pixeldrain"; + private const string ListingTypeFile = "u"; + private const string ListingTypeList = "l"; + + public static async Task ParseIndex(HttpClient httpClient, WebDirectory webDirectory, string html) + { + try + { + webDirectory = await ScanAsync(httpClient, webDirectory, html); + } + catch (Exception ex) + { + Logger.Error(ex, $"Error parsing {Parser} for URL: {webDirectory.Url}"); + webDirectory.Error = true; + + OpenDirectoryIndexer.Session.Errors++; + + if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) + { + OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); + } + + throw; + } + + return webDirectory; + } + + private static async Task ScanAsync(HttpClient httpClient, WebDirectory webDirectory, string html) + { + Logger.Debug($"Retrieving listings for {webDirectory.Uri}"); + + webDirectory.Parser = Parser; + + try + { + Match listingTypeRegexMatch = ListingTypeRegex.Match(webDirectory.Url); + + if (!listingTypeRegexMatch.Success) + { + throw new Exception("Error determining listing type"); + } + + string listingType = listingTypeRegexMatch.Groups["ListingType"].Value; + + Logger.Warn($"Retrieving listings for {webDirectory.Uri}"); + + Match listingRegexMatch = ListingRegex.Match(html); + + webDirectory.ParsedSuccessfully = listingRegexMatch.Success; + + if (!listingRegexMatch.Success) + { + throw new Exception("Listing not found"); + } + + string responseJson = listingRegexMatch.Groups["Listing"].Value; + + if (listingType == ListingTypeList) + { + PixeldrainListResult indexResponse = PixeldrainListResult.FromJson(responseJson); + + foreach (File file in indexResponse.ApiResponse.Files) + { + webDirectory.Files.Add(new WebFile + { + Url = $"https://pixeldrain.com/api/file/{Uri.EscapeDataString(file.Id)}?download", + FileName = file.Name, + FileSize = file.Size + }); + } + } + else if (listingType == ListingTypeFile) + { + PixeldrainFileResult indexResponse = PixeldrainFileResult.FromJson(responseJson); + + webDirectory.Files.Add(new WebFile + { + Url = $"https://pixeldrain.com/api/file/{Uri.EscapeDataString(indexResponse.ApiResponse.Id)}?download", + FileName = indexResponse.ApiResponse.Name, + FileSize = indexResponse.ApiResponse.Size + }); + } + else + { + throw new Exception($"Unknown listing type '{listingType}'"); + } + } + catch (Exception ex) + { + Logger.Error(ex, $"Error processing {Parser} for URL: {webDirectory.Url}"); + webDirectory.Error = true; + + OpenDirectoryIndexer.Session.Errors++; + + if (!OpenDirectoryIndexer.Session.UrlsWithErrors.Contains(webDirectory.Url)) + { + OpenDirectoryIndexer.Session.UrlsWithErrors.Add(webDirectory.Url); + } + + //throw; + } + + return webDirectory; + } +}