tools: introduce git-change-exec tool

This new tool detects if in your git tree changed files
(compared to master branch and local-only changed files)
and uses this information to run only the specified actions.

Here it is used to run pillar's go-tests only if something
changed there, same for this tool itself and the get-deps tool.

Signed-off-by: Christoph Ostarek <[email protected]>

.github/workflows/go-tests.yml

@@ -1,3 +1,6 @@
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
 ---
 name: Go Tests
 on:  # yamllint disable-line rule:truthy

.markdownlint.yaml

@@ -1,3 +1,6 @@
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
 ---
 default: true
 line-length: false

Makefile

@@ -428,10 +428,11 @@ currentversion:
 
 test: $(LINUXKIT) test-images-patches | $(DIST)
 	@echo Running tests on $(GOMODULE)
-	make -C pkg/pillar test
+	make -C tools/git-change-exec
+	./tools/git-change-exec/git-change-exec test
+	touch pkg/pillar/results.json pkg/pillar/results.xml
 	cp pkg/pillar/results.json $(DIST)/
 	cp pkg/pillar/results.xml $(DIST)/
-	make -C eve-tools/bpftrace-compiler test
 	$(QUIET): $@: Succeeded
 
 test-profiling:

tools/git-change-exec/.gitignore

@@ -0,0 +1 @@
+git-change-exec

tools/git-change-exec/Makefile

@@ -0,0 +1,14 @@
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SOURCES=actions.go actions-lint.go actions-test.go actions_test.go main.go
+git-change-exec: $(SOURCES)
+	go build
+
+.PHONY: clean
+
+clean:
+	rm -fv git-change-exec
+
+test:$(SOURCES)
+	go test

tools/git-change-exec/README.md

@@ -0,0 +1,31 @@
+# git-change-exec
+
+This new tool detects if in your git tree files changed
+compared to:
+
+* master branch
+* stable branches
+
+also local-only files are considered.
+
+Here it is used to run pillar's go-tests only if something
+changed there, same for this tool itself and the get-deps tool.
+
+## Add new Action
+
+Open `actions.go` add new implementation of `action` `interface` and
+add it to the `actions` array.
+
+Run `git-change-exec -d` to see if your action gets triggered without
+running.
+
+## Example output
+
+```bash
+2024/09/04 09:49:17 --- running gitChangeExecTest ...
+=== RUN   TestId
+--- PASS: TestId (0.00s)
+PASS
+ok  	git-change-exec	0.004s
+2024/09/04 09:49:20 --- running gitChangeExecTest done
+```

tools/git-change-exec/actions-lint.go

@@ -0,0 +1,206 @@
+// Copyright (c) 2024 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"fmt"
+	"github.com/go-git/go-git/v5/config"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type lintSpdx struct {
+	extsMap      map[string]func(path string)
+	pathsToFix   []string
+	ownPath      string
+	organization string
+}
+
+func (s *lintSpdx) dontfix(path string) {
+	log.Printf("Cannot fix %s ...\n", path)
+}
+
+func (s *lintSpdx) copyright(commentIndicator string) []string {
+	copyrightLines := []string{
+		fmt.Sprintf("%s Copyright (c) %d %s, Inc.\n", commentIndicator, time.Now().Year(), s.organization),
+		fmt.Sprintf("%s SPDX-License-Identifier: Apache-2.0\n\n", commentIndicator),
+	}
+
+	return copyrightLines
+}
+
+func (s *lintSpdx) yamlfix(path string) {
+	log.Printf("Fixing %s ...\n", path)
+	prepend(path, s.copyright("#"))
+}
+
+func (s *lintSpdx) gofix(path string) {
+	log.Printf("Fixing %s ...\n", path)
+	prepend(path, s.copyright("//"))
+}
+
+func (s *lintSpdx) dockerfilefix(path string) {
+	log.Printf("Fixing %s ...\n", path)
+	prepend(path, s.copyright("#"))
+}
+
+func prepend(path string, license []string) {
+	backupFh, err := os.CreateTemp("/var/tmp", "git-change-exec-spdx-fix")
+	if err != nil {
+		log.Fatalf("could not create temp file: %v", err)
+	}
+	backupPath := backupFh.Name()
+	defer os.Remove(backupPath)
+
+	for _, line := range license {
+		fmt.Fprint(backupFh, line)
+	}
+
+	origFh, err := os.Open(path)
+	if err != nil {
+		log.Fatalf("could not open original file %s: %v", path, err)
+	}
+
+	_, err = io.Copy(backupFh, origFh)
+	if err != nil {
+		log.Fatalf("could not copy: %v", err)
+	}
+
+	backupFh.Close()
+	origFh.Close()
+
+	err = copyFile(backupPath, path)
+	if err != nil {
+		log.Fatalf("could not rename %s -> %s: %v", backupPath, path, err)
+	}
+
+}
+
+func readGitConfigOrganization() string {
+	cfg, err := config.LoadConfig(config.GlobalScope)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, sec := range cfg.Raw.Sections {
+		if sec.Name != "user" {
+			continue
+		}
+		// codespell:ignore
+		organizations := sec.OptionAll("organization")
+
+		for _, organization := range organizations {
+			if organization != "" {
+				return organization
+			}
+		}
+	}
+
+	return ""
+}
+
+func (s *lintSpdx) init() {
+	var err error
+
+	s.extsMap = map[string]func(path string){
+		".sh":        s.dontfix,
+		".go":        s.gofix,
+		".c":         s.dontfix,
+		".h":         s.dontfix,
+		".py":        s.dontfix,
+		".rs":        s.dontfix,
+		".yaml":      s.yamlfix,
+		".yml":       s.yamlfix,
+		"Dockerfile": s.dockerfilefix,
+	}
+
+	s.pathsToFix = make([]string, 0)
+
+	s.ownPath, err = os.Executable()
+	if err != nil {
+		log.Fatalf("could not determine executable path: %v", err)
+	}
+
+	s.organization = readGitConfigOrganization()
+
+}
+
+func (s *lintSpdx) pathMatch(path string) (func(path string), bool) {
+	f, found := s.extsMap[filepath.Ext(path)]
+	if !found {
+		f, found = s.extsMap[filepath.Base(path)]
+	}
+
+	return f, found
+}
+
+func (s *lintSpdx) hasSpdx(path string) bool {
+	scriptPath := filepath.Join(filepath.Dir(s.ownPath), "..", "spdx-check.sh")
+
+	cmd := exec.Command(scriptPath, path)
+	err := cmd.Run()
+
+	return err == nil
+}
+
+func (s *lintSpdx) match(path string) bool {
+	if s.extsMap == nil {
+		s.init()
+	}
+
+	if strings.Contains(path, "/vendor/") {
+		return false
+	}
+	_, found := s.pathMatch(path)
+	if !found {
+		return false
+	}
+	if !s.hasSpdx(path) {
+		s.pathsToFix = append(s.pathsToFix, path)
+		return true
+	}
+
+	return false
+}
+
+func (s *lintSpdx) do() error {
+	if s.organization == "" {
+		log.Printf("could not read organization from git config, cannot fix copyrights")
+		return nil
+	}
+
+	for _, path := range s.pathsToFix {
+		extFixFunc, found := s.pathMatch(path)
+		if !found {
+			continue
+		}
+
+		extFixFunc(path)
+	}
+
+	return nil
+}
+
+func copyFile(srcPath, dstPath string) error {
+	src, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	defer src.Close()
+
+	dst, err := os.Create(dstPath)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	_, err = io.Copy(dst, src)
+
+	return err
+}

tools/git-change-exec/actions-test.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2024 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+type pillarTestAction struct{}
+
+func commonIgnore(path string) bool {
+	if filepath.Ext(path) == "md" {
+		return true
+	}
+
+	if filepath.Base(path) == "README" {
+		return true
+	}
+
+	return false
+}
+
+func (b pillarTestAction) match(path string) bool {
+	if commonIgnore(path) {
+		return false
+	}
+
+	ignorePaths := []string{
+		"pkg/pillar/agentlog/cmd/",
+		"pkg/pillar/docs",
+	}
+
+	for _, ignorePath := range ignorePaths {
+		if strings.HasPrefix(path, ignorePath) {
+			return false
+		}
+	}
+	return strings.HasPrefix(path, "pkg/pillar")
+}
+
+func (b pillarTestAction) do() error {
+	return execMakeTest("pkg/pillar")
+}
+
+type getDepsTestAction struct{}
+
+func (g getDepsTestAction) match(path string) bool {
+	if commonIgnore(path) {
+		return false
+	}
+
+	return strings.HasPrefix(path, "tools/get-deps")
+
+}
+func (g getDepsTestAction) do() error {
+	return execMakeTest("tools/get-deps")
+}
+
+type gitChangeExecTest struct{}
+
+func (g gitChangeExecTest) match(path string) bool {
+	if commonIgnore(path) {
+		return false
+	}
+
+	return strings.HasPrefix(path, "tools/git-change-exec")
+
+}
+func (g gitChangeExecTest) do() error {
+	return execMakeTest("tools/git-change-exec")
+}
+
+type bpftraceCompilerExecTest struct{}
+
+func (bpftraceCompilerExecTest) match(path string) bool {
+	if commonIgnore(path) {
+		return false
+	}
+
+	return strings.HasPrefix(path, "eve-tools/bpftrace-compiler")
+
+}
+func (bpftraceCompilerExecTest) do() error {
+	return execMakeTest("eve-tools/bpftrace-compiler")
+}