- Add (programming) option to disable showing statistics

- Add GitHub rate limiting support

src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs

@@ -32,6 +32,7 @@ public class OpenDirectoryIndexer
 	private static readonly Logger HistoryLogger = LogManager.GetLogger("historyFile");
 
 	public static Session Session { get; set; }
+	public static bool ShowStatistics { get; set; } = true;
 
 	public OpenDirectoryIndexerSettings OpenDirectoryIndexerSettings { get; set; }
 
@@ -270,6 +271,17 @@ public async void StartIndexingAsync()
 			Logger.Warn("Google Drive scanning is limited to 9 directories per second!");
 		}
 
+		if (Session.Root.Uri.Host == Constants.GitHubDomain)
+		{
+			Logger.Warn("GitHub scanning has a very low rate limiting of 60 directories/requests per hour!");
+
+			if (Session.MaxThreads != 1)
+			{
+				Session.MaxThreads = 1;
+				Logger.Warn($"Reduce threads to 1 because of GitHub");
+			}
+		}
+
 		if (Session.Root.Uri.Scheme == Constants.UriScheme.Ftp || Session.Root.Uri.Scheme == Constants.UriScheme.Ftps)
 		{
 			Logger.Warn("Retrieving FTP(S) software!");
@@ -588,7 +600,7 @@ public async void StartIndexingAsync()
 				{
 					try
 					{
-						new Clipboard().SetText(Statistics.GetSessionStats(OpenDirectoryIndexer.Session, includeExtensions: true, onlyRedditStats: true));
+						new Clipboard().SetText(Statistics.GetSessionStats(Session, includeExtensions: true, onlyRedditStats: true));
 						Console.WriteLine("Copied Reddit stats to clipboard!");
 						clipboardSuccess = true;
 					}
@@ -630,6 +642,11 @@ private void SetParentDirectories(WebDirectory parent)
 
 	private void TimerStatistics_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
 	{
+		if (!ShowStatistics)
+		{
+			return;
+		}
+
 		StringBuilder stringBuilder = new StringBuilder();
 
 		if (WebDirectoriesQueue.Any() || RunningWebDirectoryThreads > 0 || WebFilesFileSizeQueue.Any() || RunningWebFileFileSizeThreads > 0)
@@ -851,7 +868,7 @@ private async Task ProcessWebDirectoryAsync(string name, WebDirectory webDirecto
 			return;
 		}
 
-		if (webDirectory.Uri.Host == Constants.GitHubApiDomain)
+		if (webDirectory.Uri.Host is Constants.GitHubDomain or Constants.GitHubApiDomain)
 		{
 			WebDirectory parsedWebDirectory = await GitHubParser.ParseIndex(HttpClient, webDirectory);
 			AddProcessedWebDirectory(webDirectory, parsedWebDirectory);

src/OpenDirectoryDownloader/Site/GitHub/GitHubParser.cs

@@ -2,9 +2,11 @@
 using NLog;
 using OpenDirectoryDownloader.Shared.Models;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Threading.Tasks;
 
 namespace OpenDirectoryDownloader.Site.GitHub;
@@ -33,11 +35,12 @@ public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDire
 				Repository = webDirectory.Uri.Segments[2].TrimEnd('/');
 
 				httpClient.DefaultRequestHeaders.Clear();
-				httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(Constants.UserAgent.Curl);
+				httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("OpenDirectoryDownloader");
 
 				Logger.Warn("Retrieving default branch");
+				HttpResponseMessage httpResponseMessage = await DoRequest(httpClient, GetApiUrl(Owner, Repository));
 
-				string json = await httpClient.GetStringAsync(GetApiUrl(Owner, Repository));
+				string json = await httpResponseMessage.Content.ReadAsStringAsync();
 				DefaultBranch = JObject.Parse(json).SelectToken("default_branch")?.Value<string>();
 
 				if (string.IsNullOrEmpty(DefaultBranch))
@@ -47,7 +50,9 @@ public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDire
 
 				Logger.Warn("Retrieving last commit SHA");
 
-				json = await httpClient.GetStringAsync($"{GetApiUrl(Owner, Repository)}/branches/{DefaultBranch}");
+				httpResponseMessage = await DoRequest(httpClient, $"{GetApiUrl(Owner, Repository)}/branches/{DefaultBranch}");
+
+				json = await httpResponseMessage.Content.ReadAsStringAsync();
 				CurrentCommitSha = JObject.Parse(json).SelectToken("commit.sha")?.Value<string>();
 
 				if (string.IsNullOrEmpty(CurrentCommitSha))
@@ -76,6 +81,59 @@ public static async Task<WebDirectory> ParseIndex(HttpClient httpClient, WebDire
 		return webDirectory;
 	}
 
+	private static string GetHeader(HttpResponseHeaders httpResponseHeaders, string headerName)
+	{
+		return httpResponseHeaders.Contains(headerName) ? httpResponseHeaders.GetValues(headerName).FirstOrDefault() : string.Empty;
+	}
+
+	private static async Task<HttpResponseMessage> DoRequest(HttpClient httpClient, string url)
+	{
+		bool rateLimit = false;
+		HttpResponseMessage httpResponseMessage;
+
+		do
+		{
+			httpResponseMessage = await httpClient.GetAsync(url);
+
+			if (httpResponseMessage.Headers.Contains("X-RateLimit-Remaining"))
+			{
+				Logger.Warn($"RateLimit remaining: {GetHeader(httpResponseMessage.Headers, "X-RateLimit-Remaining")}/{GetHeader(httpResponseMessage.Headers, "X-RateLimit-Limit")}");
+			}
+
+			if (!httpResponseMessage.IsSuccessStatusCode)
+			{
+				if (httpResponseMessage.Headers.Contains("X-RateLimit-Reset"))
+				{
+					rateLimit = true;
+					DateTimeOffset resetDateTime = Library.UnixTimestampToDateTime(long.Parse(GetHeader(httpResponseMessage.Headers, "X-RateLimit-Reset")));
+					DateTimeOffset currentDate = httpResponseMessage.Headers.Date ?? DateTimeOffset.UtcNow;
+
+					// Use Server time if possible, add 5 seconds of slack
+					TimeSpan rateLimitTimeSpan = resetDateTime - (currentDate) + TimeSpan.FromSeconds(5);
+
+					if (rateLimitTimeSpan.TotalMilliseconds < TimeSpan.FromSeconds(5).TotalMilliseconds)
+					{
+						rateLimitTimeSpan = TimeSpan.FromSeconds(5);
+					}
+
+					resetDateTime = currentDate + rateLimitTimeSpan;
+
+					Logger.Warn($"Rate limited, waiting until {resetDateTime.ToLocalTime().ToString(Constants.DateTimeFormat)}..");
+
+					OpenDirectoryIndexer.ShowStatistics = false;
+					await Task.Delay(rateLimitTimeSpan);
+					OpenDirectoryIndexer.ShowStatistics = true;
+				}
+			}
+			else
+			{
+				rateLimit = false;
+			}
+		} while (rateLimit);
+
+		return httpResponseMessage;
+	}
+
 	private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDirectory webDirectory)
 	{
 		Logger.Debug($"Retrieving listings for {webDirectory.Uri}");
@@ -99,7 +157,7 @@ private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDire
 					url = $"{GetApiUrl(Owner, Repository)}/git/trees/{treeSha}";
 				}
 
-				HttpResponseMessage httpResponseMessage = await httpClient.GetAsync(url);
+				HttpResponseMessage httpResponseMessage = await DoRequest(httpClient, url);
 
 				httpResponseMessage.EnsureSuccessStatusCode();