From ec7f288538eb6c51363b90502a30b3fb8ab6795f Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Mon, 4 Dec 2023 21:37:19 +0100 Subject: [PATCH 01/15] wip tart untested code --- pkg/tart/config.go | 53 ++++++++ pkg/tart/env.go | 71 ++++++++++ pkg/tart/environment.go | 183 +++++++++++++++++++++++++ pkg/tart/vm.go | 286 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 593 insertions(+) create mode 100644 pkg/tart/config.go create mode 100644 pkg/tart/env.go create mode 100644 pkg/tart/environment.go create mode 100644 pkg/tart/vm.go diff --git a/pkg/tart/config.go b/pkg/tart/config.go new file mode 100644 index 00000000000..cf0618d65a5 --- /dev/null +++ b/pkg/tart/config.go @@ -0,0 +1,53 @@ +package tart + +import ( + "errors" +) + +var ErrConfigFromEnvironmentFailed = errors.New("failed to load config from environment") + +const ( + // GitLab CI/CD environment adds "CUSTOM_ENV_" prefix[1] to prevent + // conflicts with system environment variables. + // + // [1]: https://docs.gitlab.com/runner/executors/custom.html#stages + envPrefixGitLabRunner = "CUSTOM_ENV_" + + // The prefix that we use to avoid confusion with Cirrus CI Cloud variables + // and remove repetition from the Config's struct declaration. + envPrefixTartExecutor = "TART_EXECUTOR_" + + // EnvTartExecutorInternalBuildsDir is an internal environment variable + // that does not use the "CUSTOM_ENV_" prefix, thus preventing the override + // by the user. + EnvTartExecutorInternalBuildsDir = "TART_EXECUTOR_INTERNAL_BUILDS_DIR" + + // EnvTartExecutorInternalCacheDir is an internal environment variable + // that does not use the "CUSTOM_ENV_" prefix, thus preventing the override + // by the user. + EnvTartExecutorInternalCacheDir = "TART_EXECUTOR_INTERNAL_CACHE_DIR" +) + +type Config struct { + SSHUsername string `env:"SSH_USERNAME" envDefault:"admin"` + SSHPassword string `env:"SSH_PASSWORD" envDefault:"admin"` + Softnet bool `env:"SOFTNET"` + Headless bool `env:"HEADLESS" envDefault:"true"` + AlwaysPull bool `env:"ALWAYS_PULL" envDefault:"true"` + HostDir bool `env:"HOST_DIR"` + Shell string `env:"SHELL"` + InstallGitlabRunner bool `env:"INSTALL_GITLAB_RUNNER"` + Timezone string `env:"TIMEZONE"` +} + +func NewConfigFromEnvironment() (Config, error) { + var config Config + + // if err := env.ParseWithOptions(&config, env.Options{ + // Prefix: envPrefixGitLabRunner + envPrefixTartExecutor, + // }); err != nil { + // return config, fmt.Errorf("%w: %v", ErrConfigFromEnvironmentFailed, err) + // } + + return config, nil +} diff --git a/pkg/tart/env.go b/pkg/tart/env.go new file mode 100644 index 00000000000..53dd1e7bc7b --- /dev/null +++ b/pkg/tart/env.go @@ -0,0 +1,71 @@ +package tart + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" +) + +var ErrGitLabEnv = errors.New("GitLab environment error") + +type Env struct { + JobID string + JobImage string + FailureExitCode int + Registry *Registry +} + +type Registry struct { + Address string + User string + Password string +} + +func (e Env) VirtualMachineID() string { + return fmt.Sprintf("gitlab-%s", e.JobID) +} + +func (e Env) HostDirPath() string { + return filepath.Join(os.TempDir(), fmt.Sprintf("tart-executor-host-dir-%s", e.JobID)) +} + +func InitEnv() (*Env, error) { + result := &Env{} + jobID, ok := os.LookupEnv("CUSTOM_ENV_CI_JOB_ID") + if !ok { + return nil, fmt.Errorf("%w: CUSTOM_ENV_CI_JOB_ID is missing", ErrGitLabEnv) + } + + result.JobID = jobID + jobImage, ok := os.LookupEnv("CUSTOM_ENV_CI_JOB_IMAGE") + if !ok { + return nil, fmt.Errorf("%w: CUSTOM_ENV_CI_JOB_IMAGE is missing", ErrGitLabEnv) + } + + result.JobImage = jobImage + failureExitCodeRaw := os.Getenv("BUILD_FAILURE_EXIT_CODE") + if failureExitCodeRaw == "" { + failureExitCodeRaw = "1" // default value + } + + failureExitCode, err := strconv.Atoi(failureExitCodeRaw) + if err != nil { + return nil, fmt.Errorf("%w: failed to parse BUILD_FAILURE_EXIT_CODE", ErrGitLabEnv) + } + result.FailureExitCode = failureExitCode + + ciRegistry, ciRegistryOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY") + ciRegistryUser, ciRegistryUserOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY_USER") + ciRegistryPassword, ciRegistryPasswordOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY_PASSWORD") + if ciRegistryOK && ciRegistryUserOK && ciRegistryPasswordOK { + result.Registry = &Registry{ + Address: ciRegistry, + User: ciRegistryUser, + Password: ciRegistryPassword, + } + } + + return result, nil +} diff --git a/pkg/tart/environment.go b/pkg/tart/environment.go new file mode 100644 index 00000000000..df2bf28cad8 --- /dev/null +++ b/pkg/tart/environment.go @@ -0,0 +1,183 @@ +package tart + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" +) + +type Environment struct { + container.HostEnvironment +} + +func Start() (container.ExecutionsEnvironment, error) { + env := &Environment{} + return env, nil +} + +func (h *Environment) ToContainerPath(path string) string { + path = h.HostEnvironment.ToContainerPath(path) + actPath := filepath.Clean(h.ActPath) + altPath := filepath.Clean(path) + if strings.HasPrefix(altPath, actPath) { + return "/Volumes/My Shared Files/act/" + altPath[len(actPath):] + } + return altPath +} + +func (e *Environment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return e.ExecWithCmdLine(command, "", env, user, workdir) +} + +func (e *Environment) ExecWithCmdLine(command []string, cmdline string, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, cmdline, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +func (e *Environment) start(ctx context.Context) error { + gitLabEnv, err := InitEnv() + if err != nil { + return err + } + + config, err := NewConfigFromEnvironment() + if err != nil { + return err + } + + if config.AlwaysPull { + log.Printf("Pulling the latest version of %s...\n", gitLabEnv.JobImage) + _, _, err := TartExecWithEnv(ctx, nil, /*additionalPullEnv(gitLabEnv.Registry)*/ + "pull", gitLabEnv.JobImage) + if err != nil { + return err + } + } + + log.Println("Cloning and configuring a new VM...") + vm, err := CreateNewVM(ctx, *gitLabEnv, 0, 0) + if err != nil { + return err + } + var customDirectoryMounts []string + err = vm.Start(config, gitLabEnv, customDirectoryMounts) + if err != nil { + return err + } + + // Monitor "tart run" command's output so it's not silenced + go vm.MonitorTartRunOutput() + + log.Println("Waiting for the VM to boot and be SSH-able...") + ssh, err := vm.OpenSSH(ctx, config) + if err != nil { + return err + } + defer ssh.Close() + + log.Println("Was able to SSH!") + return nil +} +func (e *Environment) stop(ctx context.Context) error { + gitLabEnv, err := InitEnv() + if err != nil { + return err + } + + vm := ExistingVM(*gitLabEnv) + + if err = vm.Stop(); err != nil { + log.Printf("Failed to stop VM: %v", err) + } + + if err := vm.Delete(); err != nil { + log.Printf("Failed to delete VM: %v", err) + + return err + } + + tartConfig, err := NewConfigFromEnvironment() + if err != nil { + return err + } + + if tartConfig.HostDir { + if err := os.RemoveAll(gitLabEnv.HostDirPath()); err != nil { + log.Printf("Failed to clean up %q (temporary directory from the host): %v", + gitLabEnv.HostDirPath(), err) + + return err + } + } + + return nil +} + +func (e *Environment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error { + // var wd string + // if workdir != "" { + // if filepath.IsAbs(workdir) { + // wd = workdir + // } else { + // wd = filepath.Join(e.Path, workdir) + // } + // } else { + // wd = e.Path + // } + gitLabEnv, err := InitEnv() + if err != nil { + return err + } + + vm := ExistingVM(*gitLabEnv) + + // Monitor "tart run" command's output so it's not silenced + go vm.MonitorTartRunOutput() + + config, err := NewConfigFromEnvironment() + if err != nil { + return err + } + + ssh, err := vm.OpenSSH(ctx, config) + if err != nil { + return err + } + defer ssh.Close() + + session, err := ssh.NewSession() + if err != nil { + return err + } + defer session.Close() + + // GitLab script ends with an `exit` command which will terminate the SSH session + session.Stdout = os.Stdout + session.Stderr = os.Stderr + + if config.Shell != "" { + err = session.Start(config.Shell) + } else { + err = session.Shell() + } + if err != nil { + return err + } + + return session.Wait() +} diff --git a/pkg/tart/vm.go b/pkg/tart/vm.go new file mode 100644 index 00000000000..283a546bffa --- /dev/null +++ b/pkg/tart/vm.go @@ -0,0 +1,286 @@ +package tart + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/avast/retry-go" + "golang.org/x/crypto/ssh" +) + +const tartCommandName = "tart" + +var ( + ErrTartNotFound = errors.New("tart command not found") + ErrTartFailed = errors.New("tart command returned non-zero exit code") + ErrVMFailed = errors.New("VM errored") +) + +type VM struct { + id string +} + +func ExistingVM(gitLabEnv Env) *VM { + return &VM{ + id: gitLabEnv.VirtualMachineID(), + } +} + +func CreateNewVM( + ctx context.Context, + gitLabEnv Env, + cpuOverride uint64, + memoryOverride uint64, +) (*VM, error) { + vm := &VM{ + id: gitLabEnv.VirtualMachineID(), + } + + if err := vm.cloneAndConfigure(ctx, gitLabEnv, cpuOverride, memoryOverride); err != nil { + return nil, fmt.Errorf("failed to clone the VM: %w", err) + } + + return vm, nil +} + +func (vm *VM) cloneAndConfigure( + ctx context.Context, + gitLabEnv Env, + cpuOverride uint64, + memoryOverride uint64, +) error { + _, _, err := TartExec(ctx, "clone", gitLabEnv.JobImage, vm.id) + if err != nil { + return err + } + + if cpuOverride != 0 { + _, _, err = TartExec(ctx, "set", "--cpu", strconv.FormatUint(cpuOverride, 10), vm.id) + if err != nil { + return err + } + } + + if memoryOverride != 0 { + _, _, err = TartExec(ctx, "set", "--memory", strconv.FormatUint(memoryOverride, 10), vm.id) + if err != nil { + return err + } + } + + return nil +} + +func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []string) error { + var runArgs = []string{"run"} + + if config.Softnet { + runArgs = append(runArgs, "--net-softnet") + } + + if config.Headless { + runArgs = append(runArgs, "--no-graphics") + } + + for _, customDirectoryMount := range customDirectoryMounts { + runArgs = append(runArgs, "--dir", customDirectoryMount) + } + + if config.HostDir { + runArgs = append(runArgs, "--dir", fmt.Sprintf("hostdir:%s", gitLabEnv.HostDirPath())) + } else if buildsDir, ok := os.LookupEnv(EnvTartExecutorInternalBuildsDir); ok { + runArgs = append(runArgs, "--dir", fmt.Sprintf("buildsdir:%s", buildsDir)) + } + + if cacheDir, ok := os.LookupEnv(EnvTartExecutorInternalCacheDir); ok { + runArgs = append(runArgs, "--dir", fmt.Sprintf("cachedir:%s", cacheDir)) + } + + runArgs = append(runArgs, vm.id) + + cmd := exec.Command(tartCommandName, runArgs...) + + outputFile, err := os.OpenFile(vm.tartRunOutputPath(), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + return err + } + + cmd.Stdout = outputFile + cmd.Stderr = outputFile + + cmd.SysProcAttr = &syscall.SysProcAttr{ + // Setsid: true, + } + + err = cmd.Start() + if err != nil { + return err + } + + return cmd.Process.Release() +} + +func (vm *VM) MonitorTartRunOutput() { + outputFile, err := os.Open(vm.tartRunOutputPath()) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to open VM's output file, "+ + "looks like the VM wasn't started in \"prepare\" step?\n") + + return + } + defer func() { + _ = outputFile.Close() + }() + + for { + n, err := io.Copy(os.Stdout, outputFile) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to display VM's output: %v\n", err) + + break + } + if n == 0 { + time.Sleep(100 * time.Millisecond) + + continue + } + } +} + +func (vm *VM) OpenSSH(ctx context.Context, config Config) (*ssh.Client, error) { + ip, err := vm.IP(ctx) + if err != nil { + return nil, err + } + addr := ip + ":22" + + var netConn net.Conn + if err := retry.Do(func() error { + dialer := net.Dialer{} + + netConn, err = dialer.DialContext(ctx, "tcp", addr) + + return err + }, retry.Context(ctx)); err != nil { + return nil, fmt.Errorf("%w: failed to connect via SSH: %v", ErrVMFailed, err) + } + + sshConfig := &ssh.ClientConfig{ + HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + return nil + }, + User: config.SSHUsername, + Auth: []ssh.AuthMethod{ + ssh.Password(config.SSHPassword), + }, + } + + sshConn, chans, reqs, err := ssh.NewClientConn(netConn, addr, sshConfig) + if err != nil { + return nil, fmt.Errorf("%w: failed to connect via SSH: %v", ErrVMFailed, err) + } + + return ssh.NewClient(sshConn, chans, reqs), nil +} + +func (vm *VM) IP(ctx context.Context) (string, error) { + stdout, _, err := TartExec(ctx, "ip", "--wait", "60", vm.id) + if err != nil { + return "", err + } + + return strings.TrimSpace(stdout), nil +} + +func (vm *VM) Stop() error { + _, _, err := TartExec(context.Background(), "stop", vm.id) + + return err +} + +func (vm *VM) Delete() error { + _, _, err := TartExec(context.Background(), "delete", vm.id) + if err != nil { + return fmt.Errorf("%w: failed to delete VM %s: %v", ErrVMFailed, vm.id, err) + } + + return nil +} + +func TartExec( + ctx context.Context, + args ...string, +) (string, string, error) { + return TartExecWithEnv(ctx, nil, args...) +} + +func TartExecWithEnv( + ctx context.Context, + env map[string]string, + args ...string, +) (string, string, error) { + cmd := exec.CommandContext(ctx, tartCommandName, args...) + + // Base environment + cmd.Env = cmd.Environ() + + // Environment overrides + for key, value := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, value)) + } + + var stdout, stderr bytes.Buffer + + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return "", "", fmt.Errorf("%w: %s command not found in PATH, make sure Tart is installed", + ErrTartNotFound, tartCommandName) + } + + if _, ok := err.(*exec.ExitError); ok { + // Tart command failed, redefine the error + // to be the Tart-specific output + err = fmt.Errorf("%w: %q", ErrTartFailed, firstNonEmptyLine(stderr.String(), stdout.String())) + } + } + + return stdout.String(), stderr.String(), err +} + +func firstNonEmptyLine(outputs ...string) string { + for _, output := range outputs { + for _, line := range strings.Split(output, "\n") { + if line != "" { + return line + } + } + } + + return "" +} + +func (vm *VM) tartRunOutputPath() string { + // GitLab Runner redefines the TMPDIR environment variable for + // custom executors[1] and cleans it up (you can check that by + // following the "cmdOpts.Dir" xrefs, so we don't need to bother + // with that ourselves. + // + //nolint:lll + // [1]: https://com/gitlab-org/gitlab-runner/-/blob/8f29a2558bd9e72bee1df34f6651db5ba48df029/executors/custom/command/command.go#L53 + return filepath.Join(os.TempDir(), fmt.Sprintf("%s-tart-run-output.log", vm.id)) +} From 0be92a80a8e8f154a697df605c60a7f7f905c734 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 01:16:26 +0200 Subject: [PATCH 02/15] more tests --- go.mod | 1 + go.sum | 2 + pkg/container/host_environment.go | 4 +- pkg/runner/run_context.go | 10 ++ pkg/runner/run_context_darwin.go | 100 ++++++++++++++++++ pkg/tart/config.go | 54 +--------- pkg/tart/env.go | 46 -------- pkg/tart/environment.go | 170 +++++++++++++++++------------- pkg/tart/vm.go | 36 +++---- w.yml | 20 ++++ 10 files changed, 254 insertions(+), 189 deletions(-) create mode 100644 pkg/runner/run_context_darwin.go create mode 100644 w.yml diff --git a/go.mod b/go.mod index c9968ef544f..5b4830d65dc 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/avast/retry-go v3.0.0+incompatible github.com/cloudflare/circl v1.3.7 // indirect github.com/containerd/containerd v1.7.13 // indirect github.com/containerd/log v0.1.0 // indirect diff --git a/go.sum b/go.sum index fcfaa4eae50..e331dbdff6d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index 0103ebdb2f0..d4a9869d070 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -435,9 +435,9 @@ func goArchToActionArch(arch string) string { func goOsToActionOs(os string) string { osMapper := map[string]string{ - "linux": "Linux", + "linux": "Linux", "windows": "Windows", - "darwin": "macOS", + "darwin": "macOS", } if os, ok := osMapper[os]; ok { return os diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 8abc4353cac..6ca47df5106 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -239,6 +239,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor { Mode: 0o666, Body: "", }), + rc.JobContainer.Start(true), )(ctx) } } @@ -647,6 +648,9 @@ func (rc *RunContext) startContainer() common.Executor { if rc.IsHostEnv(ctx) { return rc.startHostEnvironment()(ctx) } + if rc.IsTartEnv(ctx) { + return rc.startTartEnvironment()(ctx) + } return rc.startJobContainer()(ctx) } } @@ -657,6 +661,12 @@ func (rc *RunContext) IsHostEnv(ctx context.Context) bool { return image == "" && strings.EqualFold(platform, "-self-hosted") } +func (rc *RunContext) IsTartEnv(ctx context.Context) bool { + platform := rc.runsOnImage(ctx) + image := rc.containerImage(ctx) + return image == "" && strings.HasPrefix(platform, "tart://") +} + func (rc *RunContext) stopContainer() common.Executor { return rc.stopJobContainer() } diff --git a/pkg/runner/run_context_darwin.go b/pkg/runner/run_context_darwin.go new file mode 100644 index 00000000000..f1e3420cb49 --- /dev/null +++ b/pkg/runner/run_context_darwin.go @@ -0,0 +1,100 @@ +package runner + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" + "github.com/nektos/act/pkg/tart" +) + +func (rc *RunContext) startTartEnvironment() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0o777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0o777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0o777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &tart.Environment{ + HostEnvironment: container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + }, + Config: tart.Config{ + SSHUsername: "admin", + SSHPassword: "admin", + Softnet: false, + Headless: true, + AlwaysPull: false, + }, + Env: &tart.Env{ + JobImage: "ghcr.io/cirruslabs/macos-sonoma-base:latest", + JobID: "43", + }, + Miscpath: miscpath, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + for k, v := range rc.JobContainer.GetRunnerContext(ctx) { + if v, ok := v.(string); ok { + rc.Env[fmt.Sprintf("RUNNER_%s", strings.ToUpper(k))] = v + } + } + // for _, env := range os.Environ() { + // if k, v, ok := strings.Cut(env, "="); ok { + // // don't override + // if _, ok := rc.Env[k]; !ok { + // rc.Env[k] = v + // } + // } + // } + + return common.NewPipelineExecutor( + // rc.JobContainer.Remove(), + rc.JobContainer.Start(false), + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0o644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0o666, + Body: "", + }), + )(ctx) + } +} diff --git a/pkg/tart/config.go b/pkg/tart/config.go index cf0618d65a5..861f8d2756f 100644 --- a/pkg/tart/config.go +++ b/pkg/tart/config.go @@ -1,53 +1,9 @@ package tart -import ( - "errors" -) - -var ErrConfigFromEnvironmentFailed = errors.New("failed to load config from environment") - -const ( - // GitLab CI/CD environment adds "CUSTOM_ENV_" prefix[1] to prevent - // conflicts with system environment variables. - // - // [1]: https://docs.gitlab.com/runner/executors/custom.html#stages - envPrefixGitLabRunner = "CUSTOM_ENV_" - - // The prefix that we use to avoid confusion with Cirrus CI Cloud variables - // and remove repetition from the Config's struct declaration. - envPrefixTartExecutor = "TART_EXECUTOR_" - - // EnvTartExecutorInternalBuildsDir is an internal environment variable - // that does not use the "CUSTOM_ENV_" prefix, thus preventing the override - // by the user. - EnvTartExecutorInternalBuildsDir = "TART_EXECUTOR_INTERNAL_BUILDS_DIR" - - // EnvTartExecutorInternalCacheDir is an internal environment variable - // that does not use the "CUSTOM_ENV_" prefix, thus preventing the override - // by the user. - EnvTartExecutorInternalCacheDir = "TART_EXECUTOR_INTERNAL_CACHE_DIR" -) - type Config struct { - SSHUsername string `env:"SSH_USERNAME" envDefault:"admin"` - SSHPassword string `env:"SSH_PASSWORD" envDefault:"admin"` - Softnet bool `env:"SOFTNET"` - Headless bool `env:"HEADLESS" envDefault:"true"` - AlwaysPull bool `env:"ALWAYS_PULL" envDefault:"true"` - HostDir bool `env:"HOST_DIR"` - Shell string `env:"SHELL"` - InstallGitlabRunner bool `env:"INSTALL_GITLAB_RUNNER"` - Timezone string `env:"TIMEZONE"` -} - -func NewConfigFromEnvironment() (Config, error) { - var config Config - - // if err := env.ParseWithOptions(&config, env.Options{ - // Prefix: envPrefixGitLabRunner + envPrefixTartExecutor, - // }); err != nil { - // return config, fmt.Errorf("%w: %v", ErrConfigFromEnvironmentFailed, err) - // } - - return config, nil + SSHUsername string + SSHPassword string + Softnet bool + Headless bool + AlwaysPull bool } diff --git a/pkg/tart/env.go b/pkg/tart/env.go index 53dd1e7bc7b..d0bd8d3ac75 100644 --- a/pkg/tart/env.go +++ b/pkg/tart/env.go @@ -3,9 +3,6 @@ package tart import ( "errors" "fmt" - "os" - "path/filepath" - "strconv" ) var ErrGitLabEnv = errors.New("GitLab environment error") @@ -26,46 +23,3 @@ type Registry struct { func (e Env) VirtualMachineID() string { return fmt.Sprintf("gitlab-%s", e.JobID) } - -func (e Env) HostDirPath() string { - return filepath.Join(os.TempDir(), fmt.Sprintf("tart-executor-host-dir-%s", e.JobID)) -} - -func InitEnv() (*Env, error) { - result := &Env{} - jobID, ok := os.LookupEnv("CUSTOM_ENV_CI_JOB_ID") - if !ok { - return nil, fmt.Errorf("%w: CUSTOM_ENV_CI_JOB_ID is missing", ErrGitLabEnv) - } - - result.JobID = jobID - jobImage, ok := os.LookupEnv("CUSTOM_ENV_CI_JOB_IMAGE") - if !ok { - return nil, fmt.Errorf("%w: CUSTOM_ENV_CI_JOB_IMAGE is missing", ErrGitLabEnv) - } - - result.JobImage = jobImage - failureExitCodeRaw := os.Getenv("BUILD_FAILURE_EXIT_CODE") - if failureExitCodeRaw == "" { - failureExitCodeRaw = "1" // default value - } - - failureExitCode, err := strconv.Atoi(failureExitCodeRaw) - if err != nil { - return nil, fmt.Errorf("%w: failed to parse BUILD_FAILURE_EXIT_CODE", ErrGitLabEnv) - } - result.FailureExitCode = failureExitCode - - ciRegistry, ciRegistryOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY") - ciRegistryUser, ciRegistryUserOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY_USER") - ciRegistryPassword, ciRegistryPasswordOK := os.LookupEnv("CUSTOM_ENV_CI_REGISTRY_PASSWORD") - if ciRegistryOK && ciRegistryUserOK && ciRegistryPasswordOK { - result.Registry = &Registry{ - Address: ciRegistry, - User: ciRegistryUser, - Password: ciRegistryPassword, - } - } - - return result, nil -} diff --git a/pkg/tart/environment.go b/pkg/tart/environment.go index df2bf28cad8..936300c5617 100644 --- a/pkg/tart/environment.go +++ b/pkg/tart/environment.go @@ -3,30 +3,41 @@ package tart import ( "context" "fmt" + "io" "log" "os" "path/filepath" "strings" + "github.com/kballard/go-shellquote" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" ) type Environment struct { container.HostEnvironment + vm *VM + Config Config + Env *Env + Miscpath string } -func Start() (container.ExecutionsEnvironment, error) { - env := &Environment{} - return env, nil +// "/Volumes/My Shared Files/act/" +func (e *Environment) ToHostPath(path string) string { + actPath := filepath.Clean("/private/tmp/act/") + altPath := filepath.Clean(path) + if strings.HasPrefix(altPath, actPath) { + return e.Miscpath + altPath[len(actPath):] + } + return altPath } -func (h *Environment) ToContainerPath(path string) string { - path = h.HostEnvironment.ToContainerPath(path) - actPath := filepath.Clean(h.ActPath) +func (e *Environment) ToContainerPath(path string) string { + path = e.HostEnvironment.ToContainerPath(path) + actPath := filepath.Clean(e.Miscpath) altPath := filepath.Clean(path) if strings.HasPrefix(altPath, actPath) { - return "/Volumes/My Shared Files/act/" + altPath[len(actPath):] + return "/private/tmp/act/" + altPath[len(actPath):] } return altPath } @@ -49,20 +60,20 @@ func (e *Environment) ExecWithCmdLine(command []string, cmdline string, env map[ } } +func (e *Environment) Start(b bool) common.Executor { + return e.HostEnvironment.Start(b).Then(func(ctx context.Context) error { + return e.start(ctx) + }) +} + func (e *Environment) start(ctx context.Context) error { - gitLabEnv, err := InitEnv() - if err != nil { - return err - } + gitLabEnv := e.Env - config, err := NewConfigFromEnvironment() - if err != nil { - return err - } + config := e.Config if config.AlwaysPull { log.Printf("Pulling the latest version of %s...\n", gitLabEnv.JobImage) - _, _, err := TartExecWithEnv(ctx, nil, /*additionalPullEnv(gitLabEnv.Registry)*/ + _, _, err := ExecWithEnv(ctx, nil, /*additionalPullEnv(gitLabEnv.Registry)*/ "pull", gitLabEnv.JobImage) if err != nil { return err @@ -75,33 +86,23 @@ func (e *Environment) start(ctx context.Context) error { return err } var customDirectoryMounts []string + os.MkdirAll(e.Miscpath, 0666) + customDirectoryMounts = append(customDirectoryMounts, "act:"+e.Miscpath) err = vm.Start(config, gitLabEnv, customDirectoryMounts) if err != nil { return err } - // Monitor "tart run" command's output so it's not silenced - go vm.MonitorTartRunOutput() - - log.Println("Waiting for the VM to boot and be SSH-able...") - ssh, err := vm.OpenSSH(ctx, config) - if err != nil { - return err - } - defer ssh.Close() - - log.Println("Was able to SSH!") - return nil + return e.execRaw(ctx, "ln -sf '/Volumes/My Shared Files/act' /private/tmp/act") } -func (e *Environment) stop(ctx context.Context) error { - gitLabEnv, err := InitEnv() - if err != nil { - return err - } +func (e *Environment) Stop(ctx context.Context) error { + log.Println("Stop VM?") + + gitLabEnv := e.Env vm := ExistingVM(*gitLabEnv) - if err = vm.Stop(); err != nil { + if err := vm.Stop(); err != nil { log.Printf("Failed to stop VM: %v", err) } @@ -111,48 +112,58 @@ func (e *Environment) stop(ctx context.Context) error { return err } - tartConfig, err := NewConfigFromEnvironment() - if err != nil { - return err - } + // tartConfig := e.Config - if tartConfig.HostDir { - if err := os.RemoveAll(gitLabEnv.HostDirPath()); err != nil { - log.Printf("Failed to clean up %q (temporary directory from the host): %v", - gitLabEnv.HostDirPath(), err) + // if tartConfig.HostDir { + // if err := os.RemoveAll(gitLabEnv.HostDirPath()); err != nil { + // log.Printf("Failed to clean up %q (temporary directory from the host): %v", + // gitLabEnv.HostDirPath(), err) - return err - } - } + // return err + // } + // } return nil } -func (e *Environment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, _, workdir string) error { - // var wd string - // if workdir != "" { - // if filepath.IsAbs(workdir) { - // wd = workdir - // } else { - // wd = filepath.Join(e.Path, workdir) - // } - // } else { - // wd = e.Path - // } - gitLabEnv, err := InitEnv() - if err != nil { - return err +func (e *Environment) Remove() common.Executor { + return func(ctx context.Context) error { + e.Stop(ctx) + log.Println("Remove VM?") + if e.CleanUp != nil { + e.CleanUp() + } + _ = os.RemoveAll(e.Path) + return e.Close()(ctx) + } +} +func (e *Environment) exec(ctx context.Context, command []string, _ string, env map[string]string, _, workdir string) error { + var wd string + if workdir != "" { + if filepath.IsAbs(workdir) { + wd = filepath.Clean(workdir) + } else { + wd = filepath.Clean(filepath.Join(e.Path, workdir)) + } + } else { + wd = e.ToContainerPath(e.Path) + } + envs := "" + for k, v := range env { + envs += shellquote.Join(k) + "=" + shellquote.Join(v) + " " } + return e.execRaw(ctx, "cd "+shellquote.Join(wd)+"\nenv "+envs+shellquote.Join(command...)+"\nexit $?") +} + +func (e *Environment) execRaw(ctx context.Context, script string) error { + gitLabEnv := e.Env vm := ExistingVM(*gitLabEnv) // Monitor "tart run" command's output so it's not silenced go vm.MonitorTartRunOutput() - config, err := NewConfigFromEnvironment() - if err != nil { - return err - } + config := e.Config ssh, err := vm.OpenSSH(ctx, config) if err != nil { @@ -166,18 +177,35 @@ func (e *Environment) exec(ctx context.Context, command []string, cmdline string } defer session.Close() - // GitLab script ends with an `exit` command which will terminate the SSH session - session.Stdout = os.Stdout - session.Stderr = os.Stderr + os.Stdout.WriteString(script + "\n") - if config.Shell != "" { - err = session.Start(config.Shell) - } else { - err = session.Shell() - } + session.Stdin = strings.NewReader( + script, + ) + session.Stdout = e.StdOut + session.Stderr = e.StdOut + + err = session.Shell() if err != nil { return err } return session.Wait() } + +func (e *Environment) GetActPath() string { + return e.ToContainerPath(e.HostEnvironment.GetActPath()) +} + +func (e *Environment) Copy(destPath string, files ...*container.FileEntry) common.Executor { + return e.HostEnvironment.Copy(e.ToHostPath(destPath), files...) +} +func (e *Environment) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error { + return e.HostEnvironment.CopyTarStream(ctx, e.ToHostPath(destPath), tarStream) +} +func (e *Environment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return e.HostEnvironment.CopyDir(e.ToHostPath(destPath), srcPath, useGitIgnore) +} +func (e *Environment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + return e.HostEnvironment.GetContainerArchive(ctx, e.ToHostPath(srcPath)) +} diff --git a/pkg/tart/vm.go b/pkg/tart/vm.go index 283a546bffa..c3952dad850 100644 --- a/pkg/tart/vm.go +++ b/pkg/tart/vm.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log" "net" "os" "os/exec" @@ -43,6 +44,7 @@ func CreateNewVM( cpuOverride uint64, memoryOverride uint64, ) (*VM, error) { + log.Print("CreateNewVM") vm := &VM{ id: gitLabEnv.VirtualMachineID(), } @@ -60,20 +62,20 @@ func (vm *VM) cloneAndConfigure( cpuOverride uint64, memoryOverride uint64, ) error { - _, _, err := TartExec(ctx, "clone", gitLabEnv.JobImage, vm.id) + _, _, err := Exec(ctx, "clone", gitLabEnv.JobImage, vm.id) if err != nil { return err } if cpuOverride != 0 { - _, _, err = TartExec(ctx, "set", "--cpu", strconv.FormatUint(cpuOverride, 10), vm.id) + _, _, err = Exec(ctx, "set", "--cpu", strconv.FormatUint(cpuOverride, 10), vm.id) if err != nil { return err } } if memoryOverride != 0 { - _, _, err = TartExec(ctx, "set", "--memory", strconv.FormatUint(memoryOverride, 10), vm.id) + _, _, err = Exec(ctx, "set", "--memory", strconv.FormatUint(memoryOverride, 10), vm.id) if err != nil { return err } @@ -83,6 +85,7 @@ func (vm *VM) cloneAndConfigure( } func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []string) error { + os.Remove(vm.tartRunOutputPath()) var runArgs = []string{"run"} if config.Softnet { @@ -97,16 +100,6 @@ func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []strin runArgs = append(runArgs, "--dir", customDirectoryMount) } - if config.HostDir { - runArgs = append(runArgs, "--dir", fmt.Sprintf("hostdir:%s", gitLabEnv.HostDirPath())) - } else if buildsDir, ok := os.LookupEnv(EnvTartExecutorInternalBuildsDir); ok { - runArgs = append(runArgs, "--dir", fmt.Sprintf("buildsdir:%s", buildsDir)) - } - - if cacheDir, ok := os.LookupEnv(EnvTartExecutorInternalCacheDir); ok { - runArgs = append(runArgs, "--dir", fmt.Sprintf("cachedir:%s", cacheDir)) - } - runArgs = append(runArgs, vm.id) cmd := exec.Command(tartCommandName, runArgs...) @@ -115,12 +108,13 @@ func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []strin if err != nil { return err } + outputFile.WriteString(strings.Join(runArgs, " ") + "\n") cmd.Stdout = outputFile cmd.Stderr = outputFile cmd.SysProcAttr = &syscall.SysProcAttr{ - // Setsid: true, + Setsid: true, } err = cmd.Start() @@ -177,7 +171,7 @@ func (vm *VM) OpenSSH(ctx context.Context, config Config) (*ssh.Client, error) { } sshConfig := &ssh.ClientConfig{ - HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + HostKeyCallback: func(_ string, _ net.Addr, _ ssh.PublicKey) error { return nil }, User: config.SSHUsername, @@ -195,7 +189,7 @@ func (vm *VM) OpenSSH(ctx context.Context, config Config) (*ssh.Client, error) { } func (vm *VM) IP(ctx context.Context) (string, error) { - stdout, _, err := TartExec(ctx, "ip", "--wait", "60", vm.id) + stdout, _, err := Exec(ctx, "ip", "--wait", "60", vm.id) if err != nil { return "", err } @@ -204,13 +198,13 @@ func (vm *VM) IP(ctx context.Context) (string, error) { } func (vm *VM) Stop() error { - _, _, err := TartExec(context.Background(), "stop", vm.id) + _, _, err := Exec(context.Background(), "stop", vm.id) return err } func (vm *VM) Delete() error { - _, _, err := TartExec(context.Background(), "delete", vm.id) + _, _, err := Exec(context.Background(), "delete", vm.id) if err != nil { return fmt.Errorf("%w: failed to delete VM %s: %v", ErrVMFailed, vm.id, err) } @@ -218,14 +212,14 @@ func (vm *VM) Delete() error { return nil } -func TartExec( +func Exec( ctx context.Context, args ...string, ) (string, string, error) { - return TartExecWithEnv(ctx, nil, args...) + return ExecWithEnv(ctx, nil, args...) } -func TartExecWithEnv( +func ExecWithEnv( ctx context.Context, env map[string]string, args ...string, diff --git a/w.yml b/w.yml new file mode 100644 index 00000000000..b5e042ee4d1 --- /dev/null +++ b/w.yml @@ -0,0 +1,20 @@ +on: push +jobs: + _: + runs-on: macos-14 + steps: + - run: uname -a + - run: | + echo "$TEST" + env: + TEST: my test env + - run: | + echo "$TEST" + env: + TEST: | + my test env + another line + - uses: actions/github-script@v6 + with: + script: console.log("Hello World") + github-token: what \ No newline at end of file From b751b5984c9511fa8fbb42871b7d14a5c5d45c35 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 12:25:43 +0200 Subject: [PATCH 03/15] fixes --- pkg/tart/environment.go | 15 +++++++++++++-- pkg/tart/vm.go | 22 ++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/pkg/tart/environment.go b/pkg/tart/environment.go index 936300c5617..d12d7eef797 100644 --- a/pkg/tart/environment.go +++ b/pkg/tart/environment.go @@ -88,6 +88,7 @@ func (e *Environment) start(ctx context.Context) error { var customDirectoryMounts []string os.MkdirAll(e.Miscpath, 0666) customDirectoryMounts = append(customDirectoryMounts, "act:"+e.Miscpath) + e.vm = vm err = vm.Start(config, gitLabEnv, customDirectoryMounts) if err != nil { return err @@ -100,7 +101,12 @@ func (e *Environment) Stop(ctx context.Context) error { gitLabEnv := e.Env - vm := ExistingVM(*gitLabEnv) + var vm *VM + if e.vm != nil { + vm = e.vm + } else { + vm = ExistingVM(*gitLabEnv) + } if err := vm.Stop(); err != nil { log.Printf("Failed to stop VM: %v", err) @@ -158,7 +164,12 @@ func (e *Environment) exec(ctx context.Context, command []string, _ string, env func (e *Environment) execRaw(ctx context.Context, script string) error { gitLabEnv := e.Env - vm := ExistingVM(*gitLabEnv) + var vm *VM + if e.vm != nil { + vm = e.vm + } else { + vm = ExistingVM(*gitLabEnv) + } // Monitor "tart run" command's output so it's not silenced go vm.MonitorTartRunOutput() diff --git a/pkg/tart/vm.go b/pkg/tart/vm.go index c3952dad850..dd2148f5eab 100644 --- a/pkg/tart/vm.go +++ b/pkg/tart/vm.go @@ -29,7 +29,8 @@ var ( ) type VM struct { - id string + id string + runcmd *exec.Cmd } func ExistingVM(gitLabEnv Env) *VM { @@ -121,8 +122,8 @@ func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []strin if err != nil { return err } - - return cmd.Process.Release() + vm.runcmd = cmd + return nil } func (vm *VM) MonitorTartRunOutput() { @@ -198,9 +199,18 @@ func (vm *VM) IP(ctx context.Context) (string, error) { } func (vm *VM) Stop() error { - _, _, err := Exec(context.Background(), "stop", vm.id) - - return err + log.Println("Stop VM REAL?") + if vm.runcmd != nil { + log.Println("send sigint?") + vm.runcmd.Process.Signal(os.Interrupt) + log.Println("wait?") + vm.runcmd.Wait() + log.Println("wait done?") + return nil + } else { + _, _, err := Exec(context.Background(), "stop", vm.id) + return err + } } func (vm *VM) Delete() error { From e4d7c63571a41a66d0d457fb7c5d7220454c370f Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 13:03:07 +0200 Subject: [PATCH 04/15] fixes --- pkg/runner/run_context_darwin.go | 23 +++++++++++++++++------ pkg/runner/run_context_other.go | 16 ++++++++++++++++ pkg/tart/environment.go | 7 +++++++ w.yml | 17 ++++++++++++++++- 4 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 pkg/runner/run_context_other.go diff --git a/pkg/runner/run_context_darwin.go b/pkg/runner/run_context_darwin.go index f1e3420cb49..6930b0f4505 100644 --- a/pkg/runner/run_context_darwin.go +++ b/pkg/runner/run_context_darwin.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -43,7 +44,10 @@ func (rc *RunContext) startTartEnvironment() common.Executor { return err } toolCache := filepath.Join(cacheDir, "tool_cache") - rc.JobContainer = &tart.Environment{ + platImage := rc.runsOnImage(ctx) + platURI, _ := url.Parse(platImage) + query := platURI.Query() + tenv := &tart.Environment{ HostEnvironment: container.HostEnvironment{ Path: path, TmpDir: runnerTmp, @@ -58,16 +62,23 @@ func (rc *RunContext) startTartEnvironment() common.Executor { Config: tart.Config{ SSHUsername: "admin", SSHPassword: "admin", - Softnet: false, - Headless: true, - AlwaysPull: false, + Softnet: query.Get("softnet") == "1", + Headless: query.Get("headless") != "0", + AlwaysPull: query.Get("pull") != "0", }, Env: &tart.Env{ - JobImage: "ghcr.io/cirruslabs/macos-sonoma-base:latest", - JobID: "43", + JobImage: platURI.Host + platURI.EscapedPath(), + JobID: rc.jobContainerName(), }, Miscpath: miscpath, } + rc.JobContainer = tenv + if query.Has("sshusername") { + tenv.Config.SSHUsername = query.Get("sshusername") + } + if query.Has("sshpassword") { + tenv.Config.SSHPassword = query.Get("sshpassword") + } rc.cleanUpJobContainer = rc.JobContainer.Remove() for k, v := range rc.JobContainer.GetRunnerContext(ctx) { if v, ok := v.(string); ok { diff --git a/pkg/runner/run_context_other.go b/pkg/runner/run_context_other.go new file mode 100644 index 00000000000..3039b33986f --- /dev/null +++ b/pkg/runner/run_context_other.go @@ -0,0 +1,16 @@ +//go:build !darwin + +package runner + +import ( + "context" + "fmt" + + "github.com/nektos/act/pkg/common" +) + +func (rc *RunContext) startTartEnvironment() common.Executor { + return func(_ context.Context) error { + return fmt.Errorf("You need macOS for tart") + } +} diff --git a/pkg/tart/environment.go b/pkg/tart/environment.go index d12d7eef797..9181fb46b09 100644 --- a/pkg/tart/environment.go +++ b/pkg/tart/environment.go @@ -220,3 +220,10 @@ func (e *Environment) CopyDir(destPath string, srcPath string, useGitIgnore bool func (e *Environment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { return e.HostEnvironment.GetContainerArchive(ctx, e.ToHostPath(srcPath)) } + +func (e *Environment) GetRunnerContext(ctx context.Context) map[string]interface{} { + rctx := e.HostEnvironment.GetRunnerContext(ctx) + rctx["temp"] = e.ToContainerPath(e.TmpDir) + rctx["tool_cache"] = e.ToContainerPath(e.ToolCache) + return rctx +} diff --git a/w.yml b/w.yml index b5e042ee4d1..d4d7900cb17 100644 --- a/w.yml +++ b/w.yml @@ -13,8 +13,23 @@ jobs: env: TEST: | my test env + echo 'test' another line + - uses: actions/github-script@v6 with: script: console.log("Hello World") - github-token: what \ No newline at end of file + github-token: what + - name: Test + run: | + env + sudo mkdir /Users/m1 + - name: Test + run: | + echo test > file + - uses: actions/cache/save@v4 + id: cache + with: + path: file + key: ${{ runner.os }} + #-${{ hashFiles('**/lockfiles') }} From 2941e252646ee6a3155e2916a7fae4b88f76aa33 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 13:05:25 +0200 Subject: [PATCH 05/15] fixes --- pkg/tart/env.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/tart/env.go b/pkg/tart/env.go index d0bd8d3ac75..a9293ea9a2b 100644 --- a/pkg/tart/env.go +++ b/pkg/tart/env.go @@ -2,7 +2,6 @@ package tart import ( "errors" - "fmt" ) var ErrGitLabEnv = errors.New("GitLab environment error") @@ -21,5 +20,5 @@ type Registry struct { } func (e Env) VirtualMachineID() string { - return fmt.Sprintf("gitlab-%s", e.JobID) + return e.JobID } From 6c9ee6b2d4e95919c3f59916eaa39e66667570d6 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 13:14:19 +0200 Subject: [PATCH 06/15] lint fixes --- pkg/runner/expression.go | 2 +- pkg/tart/environment.go | 4 ++-- pkg/tart/vm.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index cfd5d42f858..48df35f4921 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -512,7 +512,7 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod for k, v := range config.Inputs { value := nestedMapLookup(ghc.Event, "inputs", k) if value == nil { - v.Default.Decode(&value) + _ = v.Default.Decode(&value) } if v.Type == "boolean" { inputs[k] = value == "true" diff --git a/pkg/tart/environment.go b/pkg/tart/environment.go index 9181fb46b09..55b8965b32b 100644 --- a/pkg/tart/environment.go +++ b/pkg/tart/environment.go @@ -86,7 +86,7 @@ func (e *Environment) start(ctx context.Context) error { return err } var customDirectoryMounts []string - os.MkdirAll(e.Miscpath, 0666) + _ = os.MkdirAll(e.Miscpath, 0666) customDirectoryMounts = append(customDirectoryMounts, "act:"+e.Miscpath) e.vm = vm err = vm.Start(config, gitLabEnv, customDirectoryMounts) @@ -134,7 +134,7 @@ func (e *Environment) Stop(ctx context.Context) error { func (e *Environment) Remove() common.Executor { return func(ctx context.Context) error { - e.Stop(ctx) + _ = e.Stop(ctx) log.Println("Remove VM?") if e.CleanUp != nil { e.CleanUp() diff --git a/pkg/tart/vm.go b/pkg/tart/vm.go index dd2148f5eab..5b2cf3c360d 100644 --- a/pkg/tart/vm.go +++ b/pkg/tart/vm.go @@ -109,7 +109,7 @@ func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []strin if err != nil { return err } - outputFile.WriteString(strings.Join(runArgs, " ") + "\n") + _, _ = outputFile.WriteString(strings.Join(runArgs, " ") + "\n") cmd.Stdout = outputFile cmd.Stderr = outputFile @@ -202,9 +202,9 @@ func (vm *VM) Stop() error { log.Println("Stop VM REAL?") if vm.runcmd != nil { log.Println("send sigint?") - vm.runcmd.Process.Signal(os.Interrupt) + _ = vm.runcmd.Process.Signal(os.Interrupt) log.Println("wait?") - vm.runcmd.Wait() + _ = vm.runcmd.Wait() log.Println("wait done?") return nil } else { From 2b1fefe448e6a7f8413bfeed6ce3a499aa0c2283 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 14:11:51 +0200 Subject: [PATCH 07/15] fixes --- pkg/tart/{config.go => config_darwin.go} | 0 pkg/tart/{env.go => env_darwin.go} | 0 pkg/tart/{environment.go => environment_darwin.go} | 0 pkg/tart/{vm.go => vm_darwin.go} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename pkg/tart/{config.go => config_darwin.go} (100%) rename pkg/tart/{env.go => env_darwin.go} (100%) rename pkg/tart/{environment.go => environment_darwin.go} (100%) rename pkg/tart/{vm.go => vm_darwin.go} (100%) diff --git a/pkg/tart/config.go b/pkg/tart/config_darwin.go similarity index 100% rename from pkg/tart/config.go rename to pkg/tart/config_darwin.go diff --git a/pkg/tart/env.go b/pkg/tart/env_darwin.go similarity index 100% rename from pkg/tart/env.go rename to pkg/tart/env_darwin.go diff --git a/pkg/tart/environment.go b/pkg/tart/environment_darwin.go similarity index 100% rename from pkg/tart/environment.go rename to pkg/tart/environment_darwin.go diff --git a/pkg/tart/vm.go b/pkg/tart/vm_darwin.go similarity index 100% rename from pkg/tart/vm.go rename to pkg/tart/vm_darwin.go From 538618a598188a56916abdf8fa6303de20f67129 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 14:44:46 +0200 Subject: [PATCH 08/15] remove sample --- w.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 w.yml diff --git a/w.yml b/w.yml deleted file mode 100644 index d4d7900cb17..00000000000 --- a/w.yml +++ /dev/null @@ -1,35 +0,0 @@ -on: push -jobs: - _: - runs-on: macos-14 - steps: - - run: uname -a - - run: | - echo "$TEST" - env: - TEST: my test env - - run: | - echo "$TEST" - env: - TEST: | - my test env - echo 'test' - another line - - - uses: actions/github-script@v6 - with: - script: console.log("Hello World") - github-token: what - - name: Test - run: | - env - sudo mkdir /Users/m1 - - name: Test - run: | - echo test > file - - uses: actions/cache/save@v4 - id: cache - with: - path: file - key: ${{ runner.os }} - #-${{ hashFiles('**/lockfiles') }} From e18951ceefe788a2c70fd673db3edbe077562311 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 14:56:18 +0200 Subject: [PATCH 09/15] remove offtopic changes --- pkg/container/host_environment.go | 4 ++-- pkg/runner/run_context.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/container/host_environment.go b/pkg/container/host_environment.go index d4a9869d070..0103ebdb2f0 100644 --- a/pkg/container/host_environment.go +++ b/pkg/container/host_environment.go @@ -435,9 +435,9 @@ func goArchToActionArch(arch string) string { func goOsToActionOs(os string) string { osMapper := map[string]string{ - "linux": "Linux", + "linux": "Linux", "windows": "Windows", - "darwin": "macOS", + "darwin": "macOS", } if os, ok := osMapper[os]; ok { return os diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index 6ca47df5106..1303c47c2de 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -239,7 +239,6 @@ func (rc *RunContext) startHostEnvironment() common.Executor { Mode: 0o666, Body: "", }), - rc.JobContainer.Start(true), )(ctx) } } From 55127181601c59d3885ac89e31009054127c0148 Mon Sep 17 00:00:00 2001 From: Christopher Homberger Date: Sat, 24 Aug 2024 15:01:13 +0200 Subject: [PATCH 10/15] remove references to original code --- pkg/tart/env_darwin.go | 6 ------ pkg/tart/environment_darwin.go | 31 ++++++++++--------------------- pkg/tart/vm_darwin.go | 28 ++++++++++------------------ 3 files changed, 20 insertions(+), 45 deletions(-) diff --git a/pkg/tart/env_darwin.go b/pkg/tart/env_darwin.go index a9293ea9a2b..d4a2c4ac3dd 100644 --- a/pkg/tart/env_darwin.go +++ b/pkg/tart/env_darwin.go @@ -1,11 +1,5 @@ package tart -import ( - "errors" -) - -var ErrGitLabEnv = errors.New("GitLab environment error") - type Env struct { JobID string JobImage string diff --git a/pkg/tart/environment_darwin.go b/pkg/tart/environment_darwin.go index 55b8965b32b..30d97c0fd55 100644 --- a/pkg/tart/environment_darwin.go +++ b/pkg/tart/environment_darwin.go @@ -67,21 +67,21 @@ func (e *Environment) Start(b bool) common.Executor { } func (e *Environment) start(ctx context.Context) error { - gitLabEnv := e.Env + actEnv := e.Env config := e.Config if config.AlwaysPull { - log.Printf("Pulling the latest version of %s...\n", gitLabEnv.JobImage) - _, _, err := ExecWithEnv(ctx, nil, /*additionalPullEnv(gitLabEnv.Registry)*/ - "pull", gitLabEnv.JobImage) + log.Printf("Pulling the latest version of %s...\n", actEnv.JobImage) + _, _, err := ExecWithEnv(ctx, nil, + "pull", actEnv.JobImage) if err != nil { return err } } log.Println("Cloning and configuring a new VM...") - vm, err := CreateNewVM(ctx, *gitLabEnv, 0, 0) + vm, err := CreateNewVM(ctx, *actEnv, 0, 0) if err != nil { return err } @@ -89,7 +89,7 @@ func (e *Environment) start(ctx context.Context) error { _ = os.MkdirAll(e.Miscpath, 0666) customDirectoryMounts = append(customDirectoryMounts, "act:"+e.Miscpath) e.vm = vm - err = vm.Start(config, gitLabEnv, customDirectoryMounts) + err = vm.Start(config, actEnv, customDirectoryMounts) if err != nil { return err } @@ -99,13 +99,13 @@ func (e *Environment) start(ctx context.Context) error { func (e *Environment) Stop(ctx context.Context) error { log.Println("Stop VM?") - gitLabEnv := e.Env + actEnv := e.Env var vm *VM if e.vm != nil { vm = e.vm } else { - vm = ExistingVM(*gitLabEnv) + vm = ExistingVM(*actEnv) } if err := vm.Stop(); err != nil { @@ -118,17 +118,6 @@ func (e *Environment) Stop(ctx context.Context) error { return err } - // tartConfig := e.Config - - // if tartConfig.HostDir { - // if err := os.RemoveAll(gitLabEnv.HostDirPath()); err != nil { - // log.Printf("Failed to clean up %q (temporary directory from the host): %v", - // gitLabEnv.HostDirPath(), err) - - // return err - // } - // } - return nil } @@ -162,13 +151,13 @@ func (e *Environment) exec(ctx context.Context, command []string, _ string, env } func (e *Environment) execRaw(ctx context.Context, script string) error { - gitLabEnv := e.Env + actEnv := e.Env var vm *VM if e.vm != nil { vm = e.vm } else { - vm = ExistingVM(*gitLabEnv) + vm = ExistingVM(*actEnv) } // Monitor "tart run" command's output so it's not silenced diff --git a/pkg/tart/vm_darwin.go b/pkg/tart/vm_darwin.go index 5b2cf3c360d..95fcc9bcc9e 100644 --- a/pkg/tart/vm_darwin.go +++ b/pkg/tart/vm_darwin.go @@ -33,24 +33,24 @@ type VM struct { runcmd *exec.Cmd } -func ExistingVM(gitLabEnv Env) *VM { +func ExistingVM(actEnv Env) *VM { return &VM{ - id: gitLabEnv.VirtualMachineID(), + id: actEnv.VirtualMachineID(), } } func CreateNewVM( ctx context.Context, - gitLabEnv Env, + actEnv Env, cpuOverride uint64, memoryOverride uint64, ) (*VM, error) { log.Print("CreateNewVM") vm := &VM{ - id: gitLabEnv.VirtualMachineID(), + id: actEnv.VirtualMachineID(), } - if err := vm.cloneAndConfigure(ctx, gitLabEnv, cpuOverride, memoryOverride); err != nil { + if err := vm.cloneAndConfigure(ctx, actEnv, cpuOverride, memoryOverride); err != nil { return nil, fmt.Errorf("failed to clone the VM: %w", err) } @@ -59,11 +59,11 @@ func CreateNewVM( func (vm *VM) cloneAndConfigure( ctx context.Context, - gitLabEnv Env, + actEnv Env, cpuOverride uint64, memoryOverride uint64, ) error { - _, _, err := Exec(ctx, "clone", gitLabEnv.JobImage, vm.id) + _, _, err := Exec(ctx, "clone", actEnv.JobImage, vm.id) if err != nil { return err } @@ -85,7 +85,7 @@ func (vm *VM) cloneAndConfigure( return nil } -func (vm *VM) Start(config Config, gitLabEnv *Env, customDirectoryMounts []string) error { +func (vm *VM) Start(config Config, _ *Env, customDirectoryMounts []string) error { os.Remove(vm.tartRunOutputPath()) var runArgs = []string{"run"} @@ -207,10 +207,9 @@ func (vm *VM) Stop() error { _ = vm.runcmd.Wait() log.Println("wait done?") return nil - } else { - _, _, err := Exec(context.Background(), "stop", vm.id) - return err } + _, _, err := Exec(context.Background(), "stop", vm.id) + return err } func (vm *VM) Delete() error { @@ -279,12 +278,5 @@ func firstNonEmptyLine(outputs ...string) string { } func (vm *VM) tartRunOutputPath() string { - // GitLab Runner redefines the TMPDIR environment variable for - // custom executors[1] and cleans it up (you can check that by - // following the "cmdOpts.Dir" xrefs, so we don't need to bother - // with that ourselves. - // - //nolint:lll - // [1]: https://com/gitlab-org/gitlab-runner/-/blob/8f29a2558bd9e72bee1df34f6651db5ba48df029/executors/custom/command/command.go#L53 return filepath.Join(os.TempDir(), fmt.Sprintf("%s-tart-run-output.log", vm.id)) } From 22d5a80c72a02837c755566b0b6fde398bc768e4 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 28 Sep 2024 00:07:33 +0200 Subject: [PATCH 11/15] cleanup vm on failed to start vm --- pkg/tart/environment_darwin.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tart/environment_darwin.go b/pkg/tart/environment_darwin.go index 30d97c0fd55..0223ad8f52d 100644 --- a/pkg/tart/environment_darwin.go +++ b/pkg/tart/environment_darwin.go @@ -83,6 +83,7 @@ func (e *Environment) start(ctx context.Context) error { log.Println("Cloning and configuring a new VM...") vm, err := CreateNewVM(ctx, *actEnv, 0, 0) if err != nil { + _ = e.Stop(ctx) return err } var customDirectoryMounts []string @@ -91,6 +92,7 @@ func (e *Environment) start(ctx context.Context) error { e.vm = vm err = vm.Start(config, actEnv, customDirectoryMounts) if err != nil { + _ = e.Stop(ctx) return err } From 175ea58b5237c4eb62e308fdb5dcd03aae19b9c8 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 20 Oct 2024 11:51:17 +0200 Subject: [PATCH 12/15] revert unrelated change --- pkg/runner/expression.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/expression.go b/pkg/runner/expression.go index 48df35f4921..cfd5d42f858 100644 --- a/pkg/runner/expression.go +++ b/pkg/runner/expression.go @@ -512,7 +512,7 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod for k, v := range config.Inputs { value := nestedMapLookup(ghc.Event, "inputs", k) if value == nil { - _ = v.Default.Decode(&value) + v.Default.Decode(&value) } if v.Type == "boolean" { inputs[k] = value == "true" From ce5d63f56a315ee124426c97d5361e453dea99cb Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 20 Oct 2024 15:30:18 +0200 Subject: [PATCH 13/15] Test tart only supported on macOS --- pkg/runner/runner_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index d1875f31a9f..fc1c73598ea 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -359,6 +359,33 @@ func TestRunEvent(t *testing.T) { } } +func TestTartNotSupportedOnNonDarwin(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + tables := []TestJobFileInfo{} + + if runtime.GOOS != "darwin" { + platforms := map[string]string{ + "macos-14": "tart://ghcr.io/cirruslabs/macos-sonoma-base:latest", + } + + tables = append(tables, []TestJobFileInfo{ + // Shells + {workdir, "shells/sh", "push", "tart not supported", platforms, secrets}, + }...) + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, &Config{}) + }) + } +} + func TestRunEventHostEnvironment(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") From bb077bcff05a1d6f53fe5df66c0eeb1d0816ecbe Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 20 Oct 2024 15:35:25 +0200 Subject: [PATCH 14/15] Update runner_test.go --- pkg/runner/runner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index fc1c73598ea..d460e393615 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -370,7 +370,7 @@ func TestTartNotSupportedOnNonDarwin(t *testing.T) { if runtime.GOOS != "darwin" { platforms := map[string]string{ - "macos-14": "tart://ghcr.io/cirruslabs/macos-sonoma-base:latest", + "ubuntu-latest": "tart://ghcr.io/cirruslabs/macos-sonoma-base:latest", } tables = append(tables, []TestJobFileInfo{ From 706746e9b6bb5bafa4124303dec5ee4a025f9966 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 26 Oct 2024 09:24:54 +0000 Subject: [PATCH 15/15] fix test --- pkg/runner/runner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index d460e393615..2c6fc727dd4 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -375,7 +375,7 @@ func TestTartNotSupportedOnNonDarwin(t *testing.T) { tables = append(tables, []TestJobFileInfo{ // Shells - {workdir, "shells/sh", "push", "tart not supported", platforms, secrets}, + {workdir, "basic", "push", "tart not supported", platforms, secrets}, }...) }