Fix snapshot resume + op snapshot config (#6309)

* Resume snapshot download only if 206 * Delete after unzipping & new hash * Fix formatting * Fix PartialContent issue --------- Co-authored-by: Jorge Mederos <[email protected]>
NethermindEth · Nov 30, 2023 · 3d74bf9 · 3d74bf9
1 parent 0034333
commit 3d74bf9
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 13 deletions.
diff --git a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs
@@ -6,8 +6,8 @@
 using Nethermind.Init.Steps;
 using Nethermind.Logging;
 using System.IO.Compression;
+using System.Net;
 using System.Security.Cryptography;
-using Nethermind.Core;
 using Nethermind.Core.Extensions;
 
 namespace Nethermind.Init.Snapshot;
@@ -58,15 +58,21 @@ private async Task InitDbFromSnapshot(CancellationToken cancellationToken)
         string snapshotFileName = Path.Combine(snapshotConfig.SnapshotDirectory, snapshotConfig.SnapshotFileName);
         Directory.CreateDirectory(snapshotConfig.SnapshotDirectory);
 
-        await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken);
-        // schedule the snapshot file deletion, but only if the download completed
-        // otherwise leave it to resume the download later
-        using Reactive.AnonymousDisposable deleteSnapshot = new(() =>
+        while (true)
         {
-            if (_logger.IsInfo)
-                _logger.Info($"Deleting snapshot file {snapshotFileName}.");
-            File.Delete(snapshotFileName);
-        });
+            try
+            {
+                await DownloadSnapshotTo(snapshotUrl, snapshotFileName, cancellationToken);
+                break;
+            }
+            catch (IOException e)
+            {
+                if (_logger.IsError)
+                    _logger.Error($"Snapshot download failed. Retrying in 5 seconds. Error: {e}");
+                await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+            }
+            cancellationToken.ThrowIfCancellationRequested();
+        }
 
         if (snapshotConfig.Checksum is not null)
         {
@@ -88,7 +94,12 @@ private async Task InitDbFromSnapshot(CancellationToken cancellationToken)
         await ExtractSnapshotTo(snapshotFileName, dbPath, cancellationToken);
 
         if (_logger.IsInfo)
+        {
             _logger.Info("Database successfully initialized from snapshot.");
+            _logger.Info($"Deleting snapshot file {snapshotFileName}.");
+        }
+
+        File.Delete(snapshotFileName);
     }
 
     private async Task DownloadSnapshotTo(
@@ -116,9 +127,24 @@ private async Task DownloadSnapshotTo(
             (await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken))
             .EnsureSuccessStatusCode();
 
+        FileMode snapshotFileMode = FileMode.Create;
+        if (snapshotFileInfo.Exists && response.StatusCode == HttpStatusCode.PartialContent)
+        {
+            snapshotFileMode = FileMode.Append;
+        }
+        else if (response.StatusCode == HttpStatusCode.OK)
+        {
+            if (snapshotFileInfo.Exists && _logger.IsWarn)
+                _logger.Warn("Download couldn't be resumed. Starting from the beginning.");
+        }
+        else
+        {
+            throw new IOException($"Unexpected status code: {response.StatusCode}");
+        }
+
         await using Stream contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
         await using FileStream snapshotFileStream = new(
-            snapshotFileName, FileMode.Append, FileAccess.Write, FileShare.None, BufferSize, true);
+            snapshotFileName, snapshotFileMode, FileAccess.Write, FileShare.None, BufferSize, true);
 
         long totalBytesRead = snapshotFileStream.Length;
         long? totalBytesToRead = totalBytesRead + response.Content.Headers.ContentLength;

diff --git a/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg
@@ -24,8 +24,8 @@
   },
   "Snapshot": {
     "Enabled": true,
-    "DownloadUrl": "http://optimism-snapshot.nethermind.io/op-mainnet-genesis.zip",
-    "SnapshotFileName": "op-mainnet-genesis.zip",
-    "Checksum": "0xe6efe4bc27f8a7eb57f1a6c3d88199c104cc7dcffe6e5d083fbaccea508f8ed5"
+    "DownloadUrl": "http://optimism-snapshot.nethermind.io/op-mainnet-genesis-v1.zip",
+    "SnapshotFileName": "op-mainnet-genesis-v1.zip",
+    "Checksum": "0xd7e15b26175c4c924acf75c5790e75d5eaa044977ca8e1904dc62d5d0769eba3"
   }
 }