From a9c5e2f9ec94969f0b4b1dbf356b8d1b85f0fa8a Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 5 Aug 2024 17:50:06 -0500 Subject: [PATCH] Add support for headers with custom request object --- BenchmarkOctaneProject/Program.cs | 4 +- OctaneEngine/Clients/DefaultClient.cs | 8 +++ OctaneEngine/Clients/HTTPClientModule.cs | 2 +- OctaneEngine/Clients/IClient.cs | 1 + OctaneEngine/Clients/OctaneClient.cs | 10 ++- OctaneEngine/Engine.cs | 14 ++-- OctaneEngine/IEngine.cs | 2 +- OctaneEngine/OctaneRequest.cs | 16 +++++ OctaneEngine/docs/README.md | 74 +++++++++++--------- OctaneTestProject/DownloadTest.cs | 2 +- OctaneTestProject/EqualityTest.cs | 2 +- OctaneTestProject/PauseResumeTest.cs | 2 +- OctaneTester/Examples/Autofac.cs | 2 +- OctaneTester/Examples/NoLogger.cs | 2 +- OctaneTester/Program.cs | 10 ++- README.md | 89 ++++++++++++------------ 16 files changed, 146 insertions(+), 94 deletions(-) create mode 100644 OctaneEngine/OctaneRequest.cs diff --git a/BenchmarkOctaneProject/Program.cs b/BenchmarkOctaneProject/Program.cs index f6979f2..057f468 100644 --- a/BenchmarkOctaneProject/Program.cs +++ b/BenchmarkOctaneProject/Program.cs @@ -78,13 +78,13 @@ public void GlobalSetup() [Benchmark] public async Task BenchmarkOctane() { - await _OctaneEngine.DownloadFile(Url, "output0.zip", pauseTokenSource, cancelTokenSource); + await _OctaneEngine.DownloadFile(new OctaneRequest(){URL = ""}, "output0.zip", pauseTokenSource, cancelTokenSource); } [Benchmark] public async Task BenchmarkOctaneLowMemory() { - await _OctaneEngine2.DownloadFile(Url, "output1.zip", pauseTokenSource, cancelTokenSource); + await _OctaneEngine2.DownloadFile(new OctaneRequest(){URL = ""}, "output1.zip", pauseTokenSource, cancelTokenSource); } [Benchmark] public async Task BenchmarkHttpClient() diff --git a/OctaneEngine/Clients/DefaultClient.cs b/OctaneEngine/Clients/DefaultClient.cs index d545bb0..25286f7 100644 --- a/OctaneEngine/Clients/DefaultClient.cs +++ b/OctaneEngine/Clients/DefaultClient.cs @@ -75,6 +75,14 @@ public void SetArrayPool(ArrayPool pool) _memPool = pool; } + public void SetHeaders(OctaneRequest req) + { + foreach (var pair in req.Headers) + { + _httpClient.DefaultRequestHeaders.Add(pair.Key, pair.Value); + } + } + private async PooledTask CopyMessageContentToStreamWithProgressAsync(HttpResponseMessage message, Stream stream, IProgress progress) { byte[] buffer = _memPool.Rent(_config.BufferSize); diff --git a/OctaneEngine/Clients/HTTPClientModule.cs b/OctaneEngine/Clients/HTTPClientModule.cs index 3413248..aaccd44 100644 --- a/OctaneEngine/Clients/HTTPClientModule.cs +++ b/OctaneEngine/Clients/HTTPClientModule.cs @@ -30,7 +30,7 @@ protected override void Load(ContainerBuilder builder) { MaxResponseContentBufferSize = cfg.BufferSize, }; - + return _client; }).As().SingleInstance(); } diff --git a/OctaneEngine/Clients/IClient.cs b/OctaneEngine/Clients/IClient.cs index 3b27048..be4338a 100644 --- a/OctaneEngine/Clients/IClient.cs +++ b/OctaneEngine/Clients/IClient.cs @@ -39,5 +39,6 @@ public interface IClient : IDisposable public void SetMmf(MemoryMappedFile file); public void SetProgressbar(ProgressBar bar); public void SetArrayPool(ArrayPool pool); + public void SetHeaders(OctaneRequest req); public PooledTask Download(string url, (long, long) piece, CancellationToken cancellationToken, PauseToken pauseToken); } \ No newline at end of file diff --git a/OctaneEngine/Clients/OctaneClient.cs b/OctaneEngine/Clients/OctaneClient.cs index 664ef66..1c1e235 100644 --- a/OctaneEngine/Clients/OctaneClient.cs +++ b/OctaneEngine/Clients/OctaneClient.cs @@ -80,7 +80,15 @@ public void SetArrayPool(ArrayPool pool) { _memPool = pool; } - + + public void SetHeaders(OctaneRequest req) + { + foreach (var pair in req.Headers) + { + _client.DefaultRequestHeaders.Add(pair.Key, pair.Value); + } + } + public async PooledTask Download(string url,(long, long) piece, CancellationToken cancellationToken, PauseToken pauseToken) { _log.LogTrace("Sending request for range ({PieceItem1},{PieceItem2})...", piece.Item1, piece.Item2); diff --git a/OctaneEngine/Engine.cs b/OctaneEngine/Engine.cs index 213495b..92d2c6b 100644 --- a/OctaneEngine/Engine.cs +++ b/OctaneEngine/Engine.cs @@ -136,13 +136,13 @@ public void SetProxy(IWebProxy proxy) /// The output file name of the download. Use 'null' to get file name from url. /// The pause token source to use for pausing and resuming. /// The cancellation token for canceling the task. - public async Task DownloadFile(string url, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null) + public async Task DownloadFile(OctaneRequest req, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null) { - var (_length, _range) = await getFileSizeAndRangeSupport(url); + var (_length, _range) = await getFileSizeAndRangeSupport(req.URL); #region Varible Initilization var logger = _factory.CreateLogger("OctaneEngine"); - var filename = outFile ?? Path.GetFileName(new Uri(url).LocalPath); + var filename = outFile ?? Path.GetFileName(new Uri(req.URL).LocalPath); var success = false; var cancellation_token = Helpers.CreateCancellationToken(cancelTokenSource, _config, filename); var pause_token = pauseTokenSource ?? new PauseTokenSource(_factory); @@ -168,7 +168,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour logger.LogInformation($"PART SIZE: {NetworkAnalyzer.PrettySize(partSize)}"); stopwatch.Start(); - _client.SetBaseAddress(url); + _client.SetBaseAddress(req.URL); try { @@ -178,6 +178,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour if (_client.IsRangeSupported()) { var pieces = Helpers.CreatePartsList(_length, partSize, logger); + _client.SetHeaders(req); _client.SetMmf(mmf); _client.SetArrayPool(memPool); logger.LogInformation("Using Octane Client to download file."); @@ -198,7 +199,7 @@ public async Task DownloadFile(string url, string outFile = null, PauseTokenSour { await Parallel.ForEachAsync(pieces, options, async (piece, token) => { - await _client.Download(url, piece, cancellation_token, pause_token.Token); + await _client.Download(req.URL, piece, cancellation_token, pause_token.Token); Interlocked.Increment(ref tasksDone); @@ -221,7 +222,8 @@ await Parallel.ForEachAsync(pieces, options, async (piece, token) => else { logger.LogInformation("Using Default Client to download file."); - await _normalClient.Download(url, (0, 0), cancellation_token, pause_token.Token); + _normalClient.SetHeaders(req); + await _normalClient.Download(req.URL, (0, 0), cancellation_token, pause_token.Token); } success = true; diff --git a/OctaneEngine/IEngine.cs b/OctaneEngine/IEngine.cs index 0787adf..3fab20e 100644 --- a/OctaneEngine/IEngine.cs +++ b/OctaneEngine/IEngine.cs @@ -7,7 +7,7 @@ namespace OctaneEngineCore; public interface IEngine { - public Task DownloadFile(string url, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null); + public Task DownloadFile(OctaneRequest octaneRequest, string outFile = null, PauseTokenSource pauseTokenSource = null, CancellationTokenSource cancelTokenSource = null); public void SetProgressCallback(Action callback); public void SetDoneCallback(Action callback); public void SetProxy(IWebProxy proxy); diff --git a/OctaneEngine/OctaneRequest.cs b/OctaneEngine/OctaneRequest.cs new file mode 100644 index 0000000..b0cec6d --- /dev/null +++ b/OctaneEngine/OctaneRequest.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Net; + +namespace OctaneEngineCore; + +public class OctaneRequest +{ + public Dictionary Headers { get; set; } + public string URL { get; set; } + + public OctaneRequest() + { + Headers = new Dictionary(); + } +} \ No newline at end of file diff --git a/OctaneEngine/docs/README.md b/OctaneEngine/docs/README.md index 2544673..348292e 100644 --- a/OctaneEngine/docs/README.md +++ b/OctaneEngine/docs/README.md @@ -24,39 +24,49 @@ dotnet add package OctaneEngineCore # Usage ```csharp -private const string Url = "https://plugins.jetbrains.com/files/7973/281233/sonarlint-intellij-7.4.0.60471.zip?updateId=281233&pluginId=7973&family=INTELLIJ"; -private static void Main() -{ - //Logging Setup - var seriLog = new LoggerConfiguration() - .Enrich.FromLogContext() - .MinimumLevel.Verbose() - .WriteTo.File("./OctaneLog.txt") - .WriteTo.Console(theme: AnsiConsoleTheme.Sixteen) - .CreateLogger(); - var factory = LoggerFactory.Create(logging => { - logging.AddSerilog(seriLog); - }); - - //JSON Config Loading - var builder = new ConfigurationBuilder(); - builder.SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", true, true); - var configRoot = builder.Build(); - var config = new OctaneConfiguration(configRoot, factory); - - //Find Optimal number of parts - var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result; - seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts); - - seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result); - seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result); - var pauseTokenSource = new PauseTokenSource(); - var cancelTokenSource = new CancellationTokenSource(); +#region Logging Configuration +var seriLog = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Error() + .WriteTo.Async(a => a.File("./OctaneLog.txt")) + .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen)) + .CreateLogger(); + var factory = LoggerFactory.Create(logging => { + logging.AddSerilog(seriLog); + }); +#endregion + +#region Configuration Loading + var builder = new ConfigurationBuilder(); + builder.SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", true, true); + var configRoot = builder.Build(); +#endregion + +#region Find and Set optimal number of parts + //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result; + //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts); +#endregion - var octaneEngine = new Engine(factory, config); - octaneEngine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait(cancelTokenSource.Token); - +//seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result); +//seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result); +var pauseTokenSource = new PauseTokenSource(); +using var cancelTokenSource = new CancellationTokenSource(); +var containerBuilder = new ContainerBuilder(); +containerBuilder.RegisterInstance(factory).As(); +containerBuilder.RegisterInstance(configRoot).As(); +containerBuilder.AddOctane(); +var engineContainer = containerBuilder.Build(); +var engine = engineContainer.Resolve(); +engine.DownloadFile( + new OctaneRequest(){ + URL = Url, + Headers = new Dictionary(){} + }, + null, + pauseTokenSource, + cancelTokenSource +).Wait(); ``` # Benchmark diff --git a/OctaneTestProject/DownloadTest.cs b/OctaneTestProject/DownloadTest.cs index de98294..5565e62 100644 --- a/OctaneTestProject/DownloadTest.cs +++ b/OctaneTestProject/DownloadTest.cs @@ -85,7 +85,7 @@ public void DownloadFile() Assert.IsTrue(File.Exists(_outFile)); }); engine.SetProgressCallback(Console.WriteLine); - engine.DownloadFile(url, _outFile, _pauseTokenSource, _cancelTokenSource).Wait(); + engine.DownloadFile(new OctaneRequest(){URL = url}, _outFile, _pauseTokenSource, _cancelTokenSource).Wait(); } catch { diff --git a/OctaneTestProject/EqualityTest.cs b/OctaneTestProject/EqualityTest.cs index 3bc840c..61a9cfd 100644 --- a/OctaneTestProject/EqualityTest.cs +++ b/OctaneTestProject/EqualityTest.cs @@ -92,7 +92,7 @@ public void FileEqualityTest() engine.SetProgressCallback(Console.WriteLine); engine.SetProxy(null); - var t = engine.DownloadFile(url, outFile, _pauseTokenSource, _cancelTokenSource); + var t = engine.DownloadFile(new OctaneRequest(){URL = url}, outFile, _pauseTokenSource, _cancelTokenSource); t.Wait(); } } diff --git a/OctaneTestProject/PauseResumeTest.cs b/OctaneTestProject/PauseResumeTest.cs index ef8c83f..e385301 100644 --- a/OctaneTestProject/PauseResumeTest.cs +++ b/OctaneTestProject/PauseResumeTest.cs @@ -78,7 +78,7 @@ public void PauseResumeFile() engine.SetProxy(null); Parallel.Invoke( () => Action(_pauseTokenSource), - () => engine.DownloadFile(url, _outFile, _pauseTokenSource, _cancelTokenSource).Wait() + () => engine.DownloadFile(new OctaneRequest(){URL = url}, _outFile, _pauseTokenSource, _cancelTokenSource).Wait() ); } diff --git a/OctaneTester/Examples/Autofac.cs b/OctaneTester/Examples/Autofac.cs index de76c09..f3ed493 100644 --- a/OctaneTester/Examples/Autofac.cs +++ b/OctaneTester/Examples/Autofac.cs @@ -38,6 +38,6 @@ public void AutofacExample() containerBuilder.AddOctane(); var engineContainer = containerBuilder.Build(); var engine = engineContainer.Resolve(); - engine.DownloadFile("", "", pauseTokenSource, cancelTokenSource); + engine.DownloadFile(new OctaneRequest(){URL = ""}, "", pauseTokenSource, cancelTokenSource); } } \ No newline at end of file diff --git a/OctaneTester/Examples/NoLogger.cs b/OctaneTester/Examples/NoLogger.cs index 918ec08..613ddb9 100644 --- a/OctaneTester/Examples/NoLogger.cs +++ b/OctaneTester/Examples/NoLogger.cs @@ -27,6 +27,6 @@ public void NoLoggerExample() containerBuilder.AddOctane(); var engineContainer = containerBuilder.Build(); var engine = engineContainer.Resolve(); - engine.DownloadFile("", "", pauseTokenSource, cancelTokenSource); + engine.DownloadFile(new OctaneRequest(){URL = ""}, "", pauseTokenSource, cancelTokenSource); } } \ No newline at end of file diff --git a/OctaneTester/Program.cs b/OctaneTester/Program.cs index 61a7090..03a71cd 100644 --- a/OctaneTester/Program.cs +++ b/OctaneTester/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading; using Autofac; @@ -52,7 +53,14 @@ private static void Main() containerBuilder.AddOctane(); var engineContainer = containerBuilder.Build(); var engine = engineContainer.Resolve(); - engine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait(); + engine.DownloadFile(new OctaneRequest() + { + URL = Url, + Headers = new Dictionary() + { + + } + }, null, pauseTokenSource, cancelTokenSource).Wait(); } } } diff --git a/README.md b/README.md index b1bb914..348292e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,8 @@ ![C#](https://github.com/gregyjames/OctaneDownloader/actions/workflows/dotnet.yml/badge.svg) [![CodeQL](https://github.com/gregyjames/OctaneDownloader/actions/workflows/codeql-analysis.yml/badge.svg?branch=master)](https://github.com/gregyjames/OctaneDownloader/actions/workflows/codeql-analysis.yml) [![CodeFactor](https://www.codefactor.io/repository/github/gregyjames/octanedownloader/badge)](https://www.codefactor.io/repository/github/gregyjames/octanedownloader) -[![codebeat badge](https://codebeat.co/badges/9154fd6f-ac4b-4f00-8910-66488582efcd)](https://codebeat.co/projects/github-com-gregyjames-octanedownloader-master) [![NuGet latest version](https://badgen.net/nuget/v/OctaneEngineCore)](https://www.nuget.org/packages/OctaneEngineCore) -![NuGet Downloads](https://img.shields.io/nuget/dt/OctaneEngineCore) - -![alt tag](https://image.ibb.co/h2tK8v/Untitled_1.png) +![Nuget](https://img.shields.io/nuget/dt/OctaneEngineCore) A high Performance C# file downloader that asyncrounously downloads files as pieces. Made as a faster, more efficent replacement to Microsoft's WebClient.Want to see the library in action? Check out [Octane YouTube Extractor](https://github.com/gregyjames/OCTANE-YoutubeExtractor) @@ -27,47 +24,49 @@ dotnet add package OctaneEngineCore # Usage ```csharp -private const string Url = "https://plugins.jetbrains.com/files/7973/281233/sonarlint-intellij-7.4.0.60471.zip?updateId=281233&pluginId=7973&family=INTELLIJ"; - -private static void Main(){ - #region Logging Configuration - var seriLog = new LoggerConfiguration() - .Enrich.FromLogContext() - .MinimumLevel.Error() - .WriteTo.Async(a => a.File("./OctaneLog.txt")) - .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen)) - .CreateLogger(); - var factory = LoggerFactory.Create(logging => - { - logging.AddSerilog(seriLog); - }); - #endregion - - #region Configuration Loading - var builder = new ConfigurationBuilder(); - builder.SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json", true, true); - var configRoot = builder.Build(); - #endregion - - #region Find and Set optimal number of parts - //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result; - //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts); - //seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result); - //seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result); - #endregion - - var pauseTokenSource = new PauseTokenSource(); - using var cancelTokenSource = new CancellationTokenSource(); - var containerBuilder = new ContainerBuilder(); - containerBuilder.RegisterInstance(factory).As(); - containerBuilder.RegisterInstance(configRoot).As(); - containerBuilder.AddOctane(); - var engineContainer = containerBuilder.Build(); - var engine = engineContainer.Resolve(); - engine.DownloadFile(Url, null, pauseTokenSource, cancelTokenSource).Wait(); -} - +#region Logging Configuration +var seriLog = new LoggerConfiguration() + .Enrich.FromLogContext() + .MinimumLevel.Error() + .WriteTo.Async(a => a.File("./OctaneLog.txt")) + .WriteTo.Async(a => a.Console(theme: AnsiConsoleTheme.Sixteen)) + .CreateLogger(); + var factory = LoggerFactory.Create(logging => { + logging.AddSerilog(seriLog); + }); +#endregion + +#region Configuration Loading + var builder = new ConfigurationBuilder(); + builder.SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", true, true); + var configRoot = builder.Build(); +#endregion + +#region Find and Set optimal number of parts + //var optimalNumberOfParts = Engine.GetOptimalNumberOfParts(Url).Result; + //seriLog.Information("Optimal number of parts to download file: {OptimalNumberOfParts}", optimalNumberOfParts); +#endregion + +//seriLog.Information("Speed: {Result}", NetworkAnalyzer.GetCurrentNetworkSpeed().Result); +//seriLog.Information("Latency: {Result}", NetworkAnalyzer.GetCurrentNetworkLatency().Result); +var pauseTokenSource = new PauseTokenSource(); +using var cancelTokenSource = new CancellationTokenSource(); +var containerBuilder = new ContainerBuilder(); +containerBuilder.RegisterInstance(factory).As(); +containerBuilder.RegisterInstance(configRoot).As(); +containerBuilder.AddOctane(); +var engineContainer = containerBuilder.Build(); +var engine = engineContainer.Resolve(); +engine.DownloadFile( + new OctaneRequest(){ + URL = Url, + Headers = new Dictionary(){} + }, + null, + pauseTokenSource, + cancelTokenSource +).Wait(); ``` # Benchmark