diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 05671dde1..f95f21516 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -51,6 +51,7 @@ env: IMG_CONTROLLER: "neondatabase/neonvm-controller" IMG_VXLAN_CONTROLLER: "neondatabase/neonvm-vxlan-controller" IMG_RUNNER: "neondatabase/neonvm-runner" + IMG_DAEMON: "neondatabase/neonvm-daemon" IMG_KERNEL: "neondatabase/vm-kernel" IMG_SCHEDULER: "neondatabase/autoscale-scheduler" IMG_AUTOSCALER_AGENT: "neondatabase/autoscaler-agent" @@ -85,6 +86,7 @@ jobs: echo "controller=${{ env.IMG_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "runner=${{ env.IMG_RUNNER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "scheduler=${{ env.IMG_SCHEDULER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT @@ -214,6 +216,15 @@ jobs: build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} + - name: Build and push neonvm-daemon image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + push: true + file: neonvm/daemon/Dockerfile + tags: ${{ needs.tags.outputs.daemon }} + - name: Generate neonvm-controller build tags id: controller-build-tags env: @@ -301,6 +312,7 @@ jobs: neonvm-controller \ neonvm-vxlan-controller \ neonvm-runner \ + neonvm-daemon \ vm-kernel \ autoscale-scheduler \ autoscaler-agent \ diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 1a238e6a5..91bbdb470 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -112,6 +112,7 @@ jobs: IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }} IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }} IMG_RUNNER: ${{ needs.build-images.outputs.runner }} + IMG_DAEMON: ${{ needs.build-images.outputs.daemon }} IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }} IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }} diff --git a/Makefile b/Makefile index 8adb85c1f..5b13a8ff6 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ IMG_CONTROLLER ?= controller:dev IMG_VXLAN_CONTROLLER ?= vxlan-controller:dev IMG_RUNNER ?= runner:dev +IMG_DAEMON ?= daemon:dev IMG_SCHEDULER ?= autoscale-scheduler:dev IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev @@ -132,8 +133,8 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries. GOOS=linux go build -o bin/runner neonvm/runner/*.go .PHONY: bin/vm-builder -bin/vm-builder: ## Build vm-builder binary. - GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" neonvm/tools/vm-builder/main.go +bin/vm-builder: docker-build-daemon ## Build vm-builder binary. + GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO} -X main.NeonvmDaemonImage=${IMG_DAEMON}" neonvm/tools/vm-builder/main.go .PHONY: run run: fmt vet ## Run a controller from your host. @@ -147,7 +148,7 @@ lint: ## Run golangci-lint against code. # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build -docker-build: docker-build-controller docker-build-runner docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler +docker-build: docker-build-controller docker-build-runner docker-build-daemon docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler .PHONY: docker-push docker-push: docker-build ## Push docker images to docker registry @@ -182,6 +183,10 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne --file neonvm/runner/Dockerfile \ . +.PHONY: docker-build-daemon +docker-build-daemon: ## Build docker image for NeonVM daemon. + docker build -t $(IMG_DAEMON) -f neonvm/daemon/Dockerfile . + .PHONY: docker-build-vxlan-controller docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller docker build \ diff --git a/neonvm/apis/neonvm/v1/virtualmachine_types.go b/neonvm/apis/neonvm/v1/virtualmachine_types.go index 004b14867..a6f56e750 100644 --- a/neonvm/apis/neonvm/v1/virtualmachine_types.go +++ b/neonvm/apis/neonvm/v1/virtualmachine_types.go @@ -136,6 +136,11 @@ type VirtualMachineSpec struct { // +optional RunnerImage *string `json:"runnerImage,omitempty"` + // Rely on neonvm-daemon inside the VM for fractional CPU limiting + // +kubebuilder:default:=false + // +optional + DelegatedCPULimits *bool `json:"delegatedCPULimits,omitempty"` + // Enable SSH on the VM. It works only if the VM image is built using VM Builder that // has SSH support (TODO: mention VM Builder version). // +kubebuilder:default:=true diff --git a/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go b/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go index b79097248..4816cb6d8 100644 --- a/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go +++ b/neonvm/apis/neonvm/v1/zz_generated.deepcopy.go @@ -723,6 +723,11 @@ func (in *VirtualMachineSpec) DeepCopyInto(out *VirtualMachineSpec) { *out = new(string) **out = **in } + if in.DelegatedCPULimits != nil { + in, out := &in.DelegatedCPULimits, &out.DelegatedCPULimits + *out = new(bool) + **out = **in + } if in.EnableSSH != nil { in, out := &in.EnableSSH, &out.EnableSSH *out = new(bool) diff --git a/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml b/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml index 39e9b06e1..dda782bb8 100644 --- a/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml +++ b/neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml @@ -831,6 +831,11 @@ spec: type: array type: object type: object + delegatedCPULimits: + default: false + description: Rely on neonvm-daemon inside the VM for fractional CPU + limiting + type: boolean disks: description: List of disk that can be mounted by virtual machine. items: diff --git a/neonvm/controllers/vm_controller.go b/neonvm/controllers/vm_controller.go index 84b164ae2..8a1ead2fd 100644 --- a/neonvm/controllers/vm_controller.go +++ b/neonvm/controllers/vm_controller.go @@ -1420,6 +1420,8 @@ func podSpec( return nil, fmt.Errorf("marshal VM Status: %w", err) } + delegatedCPULimits := lo.FromPtr(vm.Spec.DelegatedCPULimits) + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: vm.Status.PodName, @@ -1486,7 +1488,9 @@ func podSpec( }}, Command: func() []string { cmd := []string{"runner"} - if config.UseContainerMgr || config.DisableRunnerCgroup { + if delegatedCPULimits { + cmd = append(cmd, "-delegated-cgroup") + } else if config.UseContainerMgr || config.DisableRunnerCgroup { cmd = append(cmd, "-skip-cgroup-management") } if config.DisableRunnerCgroup { @@ -1535,7 +1539,7 @@ func podSpec( MountPropagation: lo.ToPtr(corev1.MountPropagationNone), } - if config.UseContainerMgr || config.DisableRunnerCgroup { + if config.UseContainerMgr || config.DisableRunnerCgroup || delegatedCPULimits { return []corev1.VolumeMount{images} } else { // the /sys/fs/cgroup mount is only necessary if neonvm-runner has to @@ -1595,7 +1599,7 @@ func podSpec( }, } - if config.UseContainerMgr { + if config.UseContainerMgr && !delegatedCPULimits { return []corev1.Container{runner, containerMgr} } else { // Return only the runner if we aren't supposed to use container-mgr @@ -1628,7 +1632,9 @@ func podSpec( }, } - if config.UseContainerMgr { + if delegatedCPULimits { + return []corev1.Volume{images} + } else if config.UseContainerMgr { return []corev1.Volume{images, containerdSock} } else if config.DisableRunnerCgroup { return []corev1.Volume{images} @@ -1687,7 +1693,7 @@ func podSpec( // If a custom neonvm-runner image is requested, use that instead: if vm.Spec.RunnerImage != nil { pod.Spec.Containers[0].Image = *vm.Spec.RunnerImage - if config.UseContainerMgr { + if config.UseContainerMgr && !delegatedCPULimits { pod.Spec.Containers[1].Image = *vm.Spec.RunnerImage } } diff --git a/neonvm/daemon/Dockerfile b/neonvm/daemon/Dockerfile new file mode 100644 index 000000000..22ad8b0bc --- /dev/null +++ b/neonvm/daemon/Dockerfile @@ -0,0 +1,25 @@ +# Build the Go binary +FROM golang:1.21 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY neonvm/daemon/main.go neonvm/daemon/main.go + +# Build +# the GOARCH has not a default value to allow the binary be built according to the host where the command +# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO +# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, +# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /neonvmd neonvm/daemon/main.go + +FROM alpine:3.18 +COPY --from=builder /neonvmd /neonvmd diff --git a/neonvm/daemon/main.go b/neonvm/daemon/main.go new file mode 100644 index 000000000..2f3689ea3 --- /dev/null +++ b/neonvm/daemon/main.go @@ -0,0 +1,179 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "go.uber.org/zap" +) + +// the default period is 100000 (i.e. 100 milliseconds). We use 5 milliseconds here because +// running out of quota can result in stalling until the end of the period, and a shorter period +// *generally* helps keep latencies more consistent (at the cost of using more CPU for scheduling). +const cpuPeriodMicroseconds = 5000 + +func main() { + addr := flag.String("addr", "", `address to bind for HTTP requests`) + cgroup := flag.String("cgroup", "", `cgroup for CPU limits`) + flag.Parse() + + if *addr == "" { + fmt.Println("neonvm-daemon missing -addr flag") + os.Exit(1) + } + + logConfig := zap.NewProductionConfig() + logConfig.Sampling = nil // Disable sampling, which the production config enables by default. + logConfig.Level.SetLevel(zap.InfoLevel) // Only "info" level and above (i.e. not debug logs) + logger := zap.Must(logConfig.Build()).Named("neonvm-daemon") + defer logger.Sync() //nolint:errcheck // what are we gonna do, log something about it? + + logger.Info("Starting neonvm-daemon", zap.String("addr", *addr), zap.String("cgroup", *cgroup)) + + srv := cpuServer{ + cgroup: *cgroup, + } + srv.run(logger, *addr) +} + +type cpuServer struct { + cgroup string +} + +func (s *cpuServer) run(logger *zap.Logger, addr string) { + logger = logger.Named("cpu-srv") + + mux := http.NewServeMux() + mux.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + _ = r.Body.Close() + + cpu, err := s.getCPU(logger) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("%d", cpu))) + } else if r.Method == http.MethodPut { + body, err := io.ReadAll(r.Body) + if err != nil { + logger.Error("could not read request body", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + milliCPU, err := strconv.ParseUint(string(body), 10, 32) + if err != nil { + logger.Error("could not parse request body as uint32", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + s.setCPU(logger, uint32(milliCPU)) + } else { + // unknown method + w.WriteHeader(http.StatusNotFound) + } + }) + + timeout := 5 * time.Second + server := http.Server{ + Addr: addr, + Handler: mux, + ReadTimeout: timeout, + ReadHeaderTimeout: timeout, + WriteTimeout: timeout, + } + + err := server.ListenAndServe() + if err != nil { + logger.Fatal("CPU server exited with error", zap.Error(err)) + } + logger.Info("CPU server exited without error") +} + +func (s *cpuServer) cpuMaxPath() string { + return fmt.Sprintf("/sys/fs/cgroup/%s/cpu.max", s.cgroup) +} + +func (s *cpuServer) setCPU(logger *zap.Logger, milliCPU uint32) error { + path := s.cpuMaxPath() + quota := milliCPU * (cpuPeriodMicroseconds / 1000) + + fileContents := fmt.Sprintf("%d %d", quota, cpuPeriodMicroseconds) + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + logger.Error("could not open cgroup cpu.max file for writing", zap.Error(err)) + return err + } + + _, err = file.WriteString(fileContents) + if err != nil { + logger.Error("could not write to cgroup cpu.max", zap.Error(err)) + return err + } + + return nil +} + +// returns the current CPU limit, measured in milli-CPUs +func (s *cpuServer) getCPU(logger *zap.Logger) (uint32, error) { + data, err := os.ReadFile(s.cpuMaxPath()) + if err != nil { + logger.Error("could not read cgroup cpu.max", zap.Error(err)) + return 0, err + } + + cpuLimit, err := parseCgroupCPUMax(string(data)) + if err != nil { + logger.Error("could not parse cgroup cpu.max", zap.Error(err)) + return 0, err + } + + if cpuLimit.quota == nil { + // "0" isn't quite correct here (maybe it should be 1<<32 - 1), but zero is a more typical + // sentinel value, and will still produce the same results. + return 0, nil + } + return uint32(1000 * (*cpuLimit.quota) / cpuLimit.period), nil +} + +type cpuMax struct { + quota *uint64 + period uint64 +} + +func parseCgroupCPUMax(data string) (*cpuMax, error) { + // the contents of cpu.max are "$MAX $PERIOD", where: + // - $MAX is either a number of microseconds or the literal string "max" (meaning no limit), and + // - $PERIOD is a number of microseconds over which to account $MAX + arr := strings.Split(strings.Trim(string(data), "\n"), " ") + if len(arr) != 2 { + return nil, errors.New("unexpected contents of cgroup cpu.max") + } + + var quota *uint64 + if arr[0] != "max" { + q, err := strconv.ParseUint(arr[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse cpu quota: %w", err) + } + quota = &q + } + + period, err := strconv.ParseUint(arr[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("could not parse cpu period: %w", err) + } + + return &cpuMax{quota: quota, period: period}, nil +} diff --git a/neonvm/runner/main.go b/neonvm/runner/main.go index 8b648e2f9..4ebe5517d 100644 --- a/neonvm/runner/main.go +++ b/neonvm/runner/main.go @@ -613,6 +613,7 @@ type Config struct { appendKernelCmdline string skipCgroupManagement bool enableDummyCPUServer bool + delegatedCgroup bool diskCacheSettings string memoryProvider vmv1.MemoryProvider autoMovableRatio string @@ -626,6 +627,7 @@ func newConfig(logger *zap.Logger) *Config { appendKernelCmdline: "", skipCgroupManagement: false, enableDummyCPUServer: false, + delegatedCgroup: false, diskCacheSettings: "cache=none", memoryProvider: "", // Require that this is explicitly set. We'll check later. autoMovableRatio: "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later. @@ -644,6 +646,9 @@ func newConfig(logger *zap.Logger) *Config { flag.BoolVar(&cfg.enableDummyCPUServer, "enable-dummy-cpu-server", cfg.skipCgroupManagement, "Provide a CPU server (unlike -skip-cgroup-management) but don't actually do anything with it") + flag.BoolVar(&cfg.delegatedCgroup, "delegated-cgroup", + cfg.delegatedCgroup, + "Forward CPU requests to neonvm-daemon inside the VM (requires -skip-cgroup-management=false)") flag.StringVar(&cfg.diskCacheSettings, "qemu-disk-cache-settings", cfg.diskCacheSettings, "Cache settings to add to -drive args for VM disks") flag.Func("memory-provider", "Set provider for memory hotplug", cfg.memoryProvider.FlagFunc) @@ -659,7 +664,13 @@ func newConfig(logger *zap.Logger) *Config { logger.Fatal("missing required flag '-memhp-auto-movable-ratio'") } if cfg.enableDummyCPUServer && !cfg.skipCgroupManagement { - logger.Fatal("flag -enable-dummy-cpu-server requires -skip-cgroup-management") + logger.Fatal("flag '-enable-dummy-cpu-server' requires '-skip-cgroup-management'") + } + if cfg.delegatedCgroup && cfg.skipCgroupManagement { + logger.Fatal("flag '-delegated-cgroup' requires '-skip-cgroup-management'") + } + if cfg.delegatedCgroup && cfg.enableDummyCPUServer { + logger.Fatal("cannot have both '-delegated-cgroup' and '-enable-dummy-cpu-server'") } return cfg @@ -1000,7 +1011,7 @@ func runQEMU( var cgroupPath string - if !cfg.skipCgroupManagement { + if !cfg.skipCgroupManagement && !cfg.delegatedCgroup { selfCgroupPath, err := getSelfCgroupPath(logger) if err != nil { return fmt.Errorf("Failed to get self cgroup path: %w", err) @@ -1030,7 +1041,7 @@ func runQEMU( wg.Add(1) go terminateQemuOnSigterm(ctx, logger, &wg) - if !cfg.skipCgroupManagement || cfg.enableDummyCPUServer { + if !cfg.skipCgroupManagement || cfg.enableDummyCPUServer || cfg.delegatedCgroup { var callbacks cpuServerCallbacks if cfg.enableDummyCPUServer { @@ -1046,8 +1057,20 @@ func runQEMU( return nil }, } + } else if cfg.delegatedCgroup { + // cgroup IS delegated -- we're not handling it, and instead need to pass it off to + // neonvm-daemon inside the VM. + callbacks = cpuServerCallbacks{ + get: func(logger *zap.Logger) (*vmv1.MilliCPU, error) { + return getNeonvmDaemonCPU() + }, + set: func(logger *zap.Logger, cpu vmv1.MilliCPU) error { + return setNeonvmDaemonCPU(cpu) + }, + } } else { - // Standard implementation -- actually set the cgroup + // Standard implementation -- we're handling it ourselves, and QEMU is running in a + // local cgroup. callbacks = cpuServerCallbacks{ get: func(logger *zap.Logger) (*vmv1.MilliCPU, error) { return getCgroupQuota(cgroupPath) @@ -1066,7 +1089,7 @@ func runQEMU( var bin string var cmd []string - if !cfg.skipCgroupManagement { + if !cfg.skipCgroupManagement && !cfg.delegatedCgroup { bin = "cgexec" cmd = append([]string{"-g", fmt.Sprintf("cpu:%s", cgroupPath), QEMU_BIN}, qemuCmd...) } else { @@ -1489,6 +1512,74 @@ func getCgroupQuota(cgroupPath string) (*vmv1.MilliCPU, error) { return &cpu, nil } +func getNeonvmDaemonCPU() (*vmv1.MilliCPU, error) { + _, vmIP, _, err := calcIPs(defaultNetworkCIDR) + if err != nil { + return nil, fmt.Errorf("could not calculate VM IP address: %w", err) + } + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + + url := fmt.Sprintf("http://%s:25183/cpu", vmIP) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("could not build request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("could not send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return nil, fmt.Errorf("neonvm-daemon responded with status %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("could not read response body: %w", err) + } + + milliCPU, err := strconv.Atoi(string(body)) + if err != nil { + return nil, fmt.Errorf("could not parse response body: %w", err) + } + + return lo.ToPtr(vmv1.MilliCPU(milliCPU)), nil +} + +func setNeonvmDaemonCPU(cpu vmv1.MilliCPU) error { + _, vmIP, _, err := calcIPs(defaultNetworkCIDR) + if err != nil { + return fmt.Errorf("could not calculate VM IP address: %w", err) + } + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + + url := fmt.Sprintf("http://%s:25183/cpu", vmIP) + body := bytes.NewReader([]byte(fmt.Sprintf("%d", uint32(cpu)))) + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, body) + if err != nil { + return fmt.Errorf("could not build request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("could not send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("neonvm-daemon responded with status %d", resp.StatusCode) + } + + return nil +} + func terminateQemuOnSigterm(ctx context.Context, logger *zap.Logger, wg *sync.WaitGroup) { logger = logger.Named("terminate-qemu-on-sigterm") diff --git a/neonvm/tools/vm-builder/files/Dockerfile.img b/neonvm/tools/vm-builder/files/Dockerfile.img index 2c7b0be47..4ddf63566 100644 --- a/neonvm/tools/vm-builder/files/Dockerfile.img +++ b/neonvm/tools/vm-builder/files/Dockerfile.img @@ -7,6 +7,8 @@ FROM {{.RootDiskImage}} AS rootdisk USER root {{.SpecMerge}} +FROM {{.NeonvmDaemonImage}} AS vm-daemon-loader + FROM alpine:3.19 AS vm-runtime # add busybox ENV BUSYBOX_VERSION 1.35.0 @@ -72,6 +74,8 @@ RUN set -e \ quota-tools \ && /helper.move-bins.sh quota edquota quotacheck quotaoff quotaon quotastats setquota repquota tune2fs +COPY --from=vm-daemon-loader /neonvmd /neonvm/bin/neonvmd + # init scripts & configs COPY inittab /neonvm/bin/inittab COPY vminit /neonvm/bin/vminit diff --git a/neonvm/tools/vm-builder/files/inittab b/neonvm/tools/vm-builder/files/inittab index 6c509f19c..028158954 100644 --- a/neonvm/tools/vm-builder/files/inittab +++ b/neonvm/tools/vm-builder/files/inittab @@ -1,5 +1,6 @@ ::sysinit:/neonvm/bin/vminit ::once:/neonvm/bin/touch /neonvm/vmstart.allowed +::respawn:/neonvm/bin/neonvmd --addr=0.0.0.0:25183 --cgroup=/neonvm-root ::respawn:/neonvm/bin/udhcpc -t 1 -T 1 -A 1 -f -i eth0 -O 121 -O 119 -s /neonvm/bin/udhcpc.script ::respawn:/neonvm/bin/udevd ::wait:/neonvm/bin/udev-init.sh diff --git a/neonvm/tools/vm-builder/main.go b/neonvm/tools/vm-builder/main.go index cb2e437be..466feb376 100644 --- a/neonvm/tools/vm-builder/main.go +++ b/neonvm/tools/vm-builder/main.go @@ -63,7 +63,8 @@ var ( ) var ( - Version string + Version string + NeonvmDaemonImage string srcImage = flag.String("src", "", `Docker image used as source for virtual machine disk image: --src=alpine:3.19`) dstImage = flag.String("dst", "", `Docker image with resulting disk image: --dst=vm-alpine:3.19`) @@ -73,6 +74,8 @@ var ( quiet = flag.Bool("quiet", false, `Show less output from the docker build process`) forcePull = flag.Bool("pull", false, `Pull src image even if already present locally`) version = flag.Bool("version", false, `Print vm-builder version`) + + daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`) ) func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error { @@ -113,6 +116,8 @@ type TemplatesContext struct { Env []string RootDiskImage string + NeonvmDaemonImage string + SpecBuild string SpecMerge string InittabCommands []inittabCommand @@ -134,6 +139,17 @@ func main() { os.Exit(0) } + if len(*daemonImageFlag) == 0 && len(NeonvmDaemonImage) == 0 { + log.Println("neonvm-daemon image not set, needs to be explicitly passed in, or compiled with -ldflags '-X main.NeonvmDaemonImage=...'") + flag.PrintDefaults() + os.Exit(1) + } + + neonvmDaemonImage := NeonvmDaemonImage + if len(*daemonImageFlag) != 0 { + neonvmDaemonImage = *daemonImageFlag + } + if len(*srcImage) == 0 { log.Println("-src not set, see usage info:") flag.PrintDefaults() @@ -274,6 +290,8 @@ func main() { Env: imageSpec.Config.Env, RootDiskImage: *srcImage, + NeonvmDaemonImage: neonvmDaemonImage, + SpecBuild: "", // overridden below if spec != nil SpecMerge: "", // overridden below if spec != nil InittabCommands: nil, // overridden below if spec != nil