From 7d6bb935a8ab2edcab83be1515e85f54bdedfef5 Mon Sep 17 00:00:00 2001 From: "[eureka@nixos]" <57543709+eureka-cpu@users.noreply.github.com> Date: Fri, 12 Jul 2024 20:02:30 -0700 Subject: [PATCH] Setup busybox rootfs (#35) Ref #34 Create the busybox filesystem for running lasr programs. Pre-pulls the busybox image from dockerhub, streaming the image to stdout and loading it into the docker registry. This speeds up the `node-start` pre-script by having the image ready beforehand when building the container to create the `rootfs`. --- deployments/lasr_node/common.nix | 100 +++++++++++++++++++++++-------- flake.lock | 24 ++++---- 2 files changed, 86 insertions(+), 38 deletions(-) diff --git a/deployments/lasr_node/common.nix b/deployments/lasr_node/common.nix index 33b719b..560cbd6 100644 --- a/deployments/lasr_node/common.nix +++ b/deployments/lasr_node/common.nix @@ -4,17 +4,18 @@ let # Pull the PD server image from dockerhub pd-image = let + name = "pingcap/pd"; platformSha256 = { "aarch64-linux" = "sha256-+IBB5p1M8g3fLjHbF90vSSAoKUidl5cdkpTulkzlMAc="; "x86_64-linux" = "sha256-xNPJrv8y6vjAPNvn9lAkghCfRGUDiBfRCUBsEYvb49Q="; - }."${system}" or (builtins.throw "Unsupported platform, must either be arm64 or amd64 Linux: found ${system}"); + }."${system}" or (builtins.throw "Unsupported platform for docker image ${name}, must either be arm64 or amd64 Linux: found ${system}"); in pkgs.dockerTools.pullImage { - imageName = "pingcap/pd"; + imageName = name; imageDigest = "sha256:0e87d077d0fd92903e26a6ebeda633d6979380aac6fc76aa24c6a02d25a404f6"; sha256 = platformSha256; finalImageTag = "latest"; - finalImageName = "pingcap/pd"; + finalImageName = name; }; # Starts the placement driver server for TiKV. # NOTE: This is a global script, which is run by default and is only @@ -31,17 +32,18 @@ let # Pull the TiKV server image from dockerhub tikv-image = let + name = "pingcap/tikv"; platformSha256 = { "aarch64-linux" = "sha256-JbogHq9FLfm7x08xkwiDF0+YyUKRXF34vHty+ZxIZh0="; "x86_64-linux" = "sha256-udLF3mAuUU08QX2Tg/mma9uu0JdtdJuxK3R1bqdKjKk="; - }.${system} or (builtins.throw "Unsupported platform, must either be arm64 or amd64 Linux: found ${system}"); + }.${system} or (builtins.throw "Unsupported platform for docker image ${name}, must either be arm64 or amd64 Linux: found ${system}"); in pkgs.dockerTools.pullImage { - imageName = "pingcap/tikv"; + imageName = name; imageDigest = "sha256:e68889611930cc054acae5a46bee862c4078af246313b414c1e6c4671dceca63"; sha256 = platformSha256; finalImageTag = "latest"; - finalImageName = "pingcap/tikv"; + finalImageName = name; }; # Starts the TiKV server. # NOTE: This is a global script, which is run by default and is only @@ -53,6 +55,28 @@ let --data-dir="/tikv" \ --pd="http://127.0.0.1:2379" ''; + busybox-stream = + let + name = "busybox"; + tag = "latest"; + platformSha256 = { + "aarch64-linux" = "sha256-Oq9xmrdoAJvwQl9WOBkJFhacWHT9JG0B384gaHrimL8="; + "x86_64-linux" = "sha256-Oq9xmrdoAJvwQl9WOBkJFhacWHT9JG0B384gaHrimL8="; + }.${system} or (builtins.throw "Unsupported platform for docker image ${name}, must either be arm64 or amd64 Linux: found ${system}"); + busyboxImage = pkgs.dockerTools.pullImage { + imageName = name; + imageDigest = "sha256:50aa4698fa6262977cff89181b2664b99d8a56dbca847bf62f2ef04854597cf8"; + sha256 = platformSha256; + finalImageTag = tag; + finalImageName = name; + }; + in + pkgs.dockerTools.streamLayeredImage { + name = name; + tag = tag; + fromImage = busyboxImage; + config.Cmd = [ "busybox" ]; + }; # Creates the working directory, scripts & initializes the IPFS node. # NOTE: This is a global script, which is run by default and is only @@ -292,7 +316,7 @@ in block_path="/app/blocks_processed.dat" eth_rpc_url="https://u0anlnjcq5:xPYLI9OMwxRqJZqhfgEiKMeGdpVjGduGKmMCNBsu46Y@u0auvfalma-u0j1mdxq0w-rpc.us0-aws.kaleido.io/" eo_contract=0x563f0efeea703237b32ae7f66123b864f3e46a3c - compute_rpc_url=ws://localhost:9125 + compute_rpc_url=ws://localhost:9125 storage_rpc_url=ws://localhost:9126 batch_interval=180 ipfs_path="/app/tmp/kubo" @@ -336,31 +360,55 @@ in node-start = { description = "Start the lasr_node process."; after = [ "init-env.service" "ipfs-start.service" ]; - preStart = '' - if [ ! -e "/app/bin" ]; then - echo "Setting up working directory.." - mkdir -p /app/bin + preStart = + let + # nix-store paths are resolved on user login, but here we have to give the absolute paths. + docker = "${pkgs.docker}/bin/docker"; + sudo = "/run/wrappers/bin/sudo"; + tar = "/run/current-system/sw/bin/tar"; + in + '' + if [ ! -e "/app/bin" ]; then + echo "Setting up working directory.." + mkdir -p /app/bin + mkdir -p /app/payload + mkdir -p /app/base_image/busybox - cd /app - printf "${procfile.text}" > "${procfile.name}" - "${pkgs.git}"/bin/git clone https://github.com/versatus/lasr.git + cd /app + printf "${procfile.text}" > "${procfile.name}" + ${pkgs.git}/bin/git clone https://github.com/versatus/lasr.git - cd /app/bin - printf "${start-ipfs.text}" > "${start-ipfs.name}" - printf "${start-lasr.text}" > "${start-lasr.name}" - printf "${start-overmind.text}" > "${start-overmind.name}" + cd /app/base_image/busybox + mkdir --mode=0755 rootfs + ${busybox-stream} | ${docker} image load + ${docker} export $(${docker} create busybox) | ${sudo} ${tar} -xf - -C rootfs --same-owner --same-permissions - for file in ./*; do - chmod +x "$file" - done + cd /app/bin + printf "${start-ipfs.text}" > "${start-ipfs.name}" + printf "${start-lasr.text}" > "${start-lasr.name}" + printf "${start-overmind.text}" > "${start-overmind.name}" - echo "Working directory '/app' is ready." - else - echo "Working directory already exists." - fi - ''; + for file in ./*; do + chmod +x "$file" + done + + # wait for busybox to set up the root filesystem for the program env + while [ ! -e "/app/base_image/busybox/rootfs/bin" ]; do + echo "Waiting for busybox filesystem to be ready, sleeping for 2s..." + sleep 2 + done + + echo "Working directory '/app' is ready." + else + echo "Working directory already exists." + fi + ''; script = '' + # grace period for all services to finish setup + sleep 5 + source "$HOME/.bashrc" + cd /app "${pkgs.lasr_node}/bin/lasr_node" ''; wantedBy = [ "default.target" ]; diff --git a/flake.lock b/flake.lock index 128862c..f7eb6c6 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1719685792, - "narHash": "sha256-WIoVERD4AN6CmfGSRPy3mfPx2dDbRHgzP2V8z6aNbaY=", + "lastModified": 1720546058, + "narHash": "sha256-iU2yVaPIZm5vMGdlT0+57vdB/aPq/V5oZFBRwYw+HBM=", "owner": "ipetkov", "repo": "crane", - "rev": "aa5dcd0518a422dfd545d565f0d5a25971fea52a", + "rev": "2d83156f23c43598cf44e152c33a59d3892f8b29", "type": "github" }, "original": { @@ -28,11 +28,11 @@ "rust-analyzer-src": [] }, "locked": { - "lastModified": 1719815435, - "narHash": "sha256-K2xFp142onP35jcx7li10xUxNVEVRWjAdY8DSuR7Naw=", + "lastModified": 1720679322, + "narHash": "sha256-6ffU/E1I9lwDoOcaXmDLf3HEwC2aab1BGh2JQVXgwYE=", "owner": "nix-community", "repo": "fenix", - "rev": "ebfe2c639111d7e82972a12711206afaeeda2450", + "rev": "87536ab1ef7378c982cbe3455e3c0d48f1879bc5", "type": "github" }, "original": { @@ -62,11 +62,11 @@ "lasr": { "flake": false, "locked": { - "lastModified": 1719421279, - "narHash": "sha256-fvh/FViPy5ki1LZtMllagCZ1kgedaNcZFNuadg+zpNg=", + "lastModified": 1720502609, + "narHash": "sha256-umP3YuQs0gDSXqOEbeH0KwgDq4v9UHNJI4XOcePj6sQ=", "owner": "versatus", "repo": "lasr", - "rev": "f054cbab0afb9b827d725d1932057cda3b16c5f2", + "rev": "91e710b9e01aa65650c0a69aa31afd291fd75448", "type": "github" }, "original": { @@ -77,11 +77,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1719826879, - "narHash": "sha256-xs7PlULe8O1SAcs/9e/HOjeUjBrU5FNtkAF/bSEcFto=", + "lastModified": 1720657034, + "narHash": "sha256-nPhbeFdyN8yn+EXmnPcBWisoypndtQbNIhSKmAinv3E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b9014df496d5b68bf7c0145d0e9b0f529ce4f2a8", + "rev": "212defe037698e18fc9521dfe451779a8979844c", "type": "github" }, "original": {