- Really use GitHub recursive parameter

README.md

@@ -117,7 +117,7 @@ If anybody have more info how to get Cloudflare to work better, let me know!
 
 ## GitHub
 
-By default GitHub has a rate limit of 60 request per hour. You can increase this limit to 5000 per hour by creating a (personal) token:
+By default GitHub has a rate limit of 60 request per hour, which is enough for 20 repositories with less than 100.000 items. You can increase this limit to 5000 per hour by creating a (personal) token:
 
 1. Go to https://github.com/settings/tokens/new
 2. Add a name like "OpenDirectoryDownloader"

src/OpenDirectoryDownloader/OpenDirectoryIndexer.cs

@@ -910,7 +910,7 @@ private async Task ProcessWebDirectoryAsync(string name, WebDirectory webDirecto
 		if (webDirectory.Uri.Host is Constants.GitHubDomain or Constants.GitHubApiDomain)
 		{
 			WebDirectory parsedWebDirectory = await GitHubParser.ParseIndex(HttpClient, webDirectory, Session.CommandLineOptions.GitHubToken);
-			AddProcessedWebDirectory(webDirectory, parsedWebDirectory);
+			AddProcessedWebDirectory(webDirectory, parsedWebDirectory, processSubdirectories: false);
 			return;
 		}
 
@@ -1514,7 +1514,6 @@ private void AddProcessedWebDirectory(WebDirectory webDirectory, WebDirectory pa
 					if (subdirectory.Uri.Host != Constants.AmazonS3Domain &&
 						subdirectory.Uri.Host != Constants.BlitzfilesTechDomain &&
 						subdirectory.Uri.Host != Constants.DropboxDomain &&
-						!subdirectory.Uri.Host.EndsWith(Constants.GitHubDomain) &&
 						subdirectory.Uri.Host != Constants.GoogleDriveDomain &&
 						!DirectoryParser.SameHostAndDirectoryFile(Session.Root.Uri, subdirectory.Uri))
 					{

src/OpenDirectoryDownloader/Site/GitHub/GitHubParser.cs

@@ -186,26 +186,42 @@ private static async Task<WebDirectory> ScanAsync(HttpClient httpClient, WebDire
 				Logger.Warn($"GitHub response is truncated with {gitHubResult.Tree.Length} items, sadly there is no paging available..");
 			}
 
-			string path = webDirectory.ParentDirectory is not null ? $"{webDirectory.Name}/" : string.Empty;
+			WebDirectory currentWebDirectory = webDirectory;
 
+			// Yes, this code is a little complicated, but it works..
 			foreach (Tree treeItem in gitHubResult.Tree)
 			{
+				WebDirectory parentWebDirectory = currentWebDirectory;
+
 				// Like directories
 				if (treeItem.Type == "tree")
 				{
-					webDirectory.Subdirectories.Add(new WebDirectory(webDirectory)
+					while (parentWebDirectory.ParentDirectory is not null && !treeItem.Path.StartsWith(parentWebDirectory.Name))
+					{
+						parentWebDirectory = parentWebDirectory.ParentDirectory;
+					}
+
+					WebDirectory newWebDirectory = new WebDirectory(parentWebDirectory)
 					{
 						Parser = Parser,
 						Url = treeItem.Url,
 						Name = treeItem.Path
-					});
+					};
+
+					parentWebDirectory.Subdirectories.Add(newWebDirectory);
+					currentWebDirectory = newWebDirectory;
 				}
 				else
 				// Like files
 				{
-					string fileName = new Uri(new Uri($"https://raw.githubusercontent.com/{Uri.EscapeDataString(Owner)}/{Uri.EscapeDataString(Repository)}/"), Path.Combine(CurrentCommitSha, path, treeItem.Path)).ToString();
+					while (parentWebDirectory.ParentDirectory is not null && !Path.GetDirectoryName(treeItem.Path).Replace("\\", "/").StartsWith(parentWebDirectory.Name))
+					{
+						parentWebDirectory = parentWebDirectory.ParentDirectory;
+					}
+
+					string fileName = new Uri(new Uri($"https://raw.githubusercontent.com/{Uri.EscapeDataString(Owner)}/{Uri.EscapeDataString(Repository)}/"), Path.Combine(CurrentCommitSha, treeItem.Path)).ToString();
 
-					webDirectory.Files.Add(new WebFile
+					parentWebDirectory.Files.Add(new WebFile
 					{
 						Url = fileName,
 						FileName = treeItem.Path,