Add ECS fields to process & FS resources (#254)

.github/workflows/unit-test.yml

@@ -78,6 +78,9 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
       - name: Check out the repo
         uses: actions/checkout@v3
         with:

beater/cloudbeat.go

@@ -23,9 +23,8 @@ import (
 	"time"
 
 	"github.com/elastic/cloudbeat/launcher"
-	"github.com/elastic/cloudbeat/pipeline" //nolint: typecheck
+	"github.com/elastic/cloudbeat/pipeline"
 	"github.com/elastic/cloudbeat/resources/fetching"
-
 	"github.com/elastic/cloudbeat/config"
 	"github.com/elastic/cloudbeat/evaluator"
 	_ "github.com/elastic/cloudbeat/processor" // Add cloudbeat default processors.

go.mod

@@ -32,6 +32,7 @@ require (
 )
 
 require (
+	github.com/djherbis/times v1.5.0
 	github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4
 	github.com/elastic/elastic-agent-autodiscover v0.1.1
 	github.com/elastic/elastic-agent-libs v0.2.5

go.sum

@@ -424,6 +424,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
 github.com/distribution/distribution/v3 v3.0.0-20210804104954-38ab4c606ee3/go.mod h1:gt38b7cvVKazi5XkHvINNytZXgTEntyhtyM3HQz46Nk=
 github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY=
 github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684/go.mod h1:UfCu3YXJJCI+IdnqGgYP82dk2+Joxmv+mUTVBES6wac=
+github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU=
+github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0=
 github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
 github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=

resources/fetchers/ecr_fetcher.go

@@ -121,3 +121,5 @@ func (res ECRResource) GetMetadata() fetching.ResourceMetadata {
 		Name:    "AWS repositories",
 	}
 }
+
+func (res ECRResource) GetElasticCommonData() any { return nil }

resources/fetchers/eks_fetcher.go

@@ -74,3 +74,5 @@ func (r EKSResource) GetMetadata() fetching.ResourceMetadata {
 		Name:    *r.Cluster.Name,
 	}
 }
+
+func (r EKSResource) GetElasticCommonData() any { return nil }

resources/fetchers/elb_fetcher.go

@@ -112,3 +112,4 @@ func (r ELBResource) GetMetadata() fetching.ResourceMetadata {
 		Name:    "",
 	}
 }
+func (r ELBResource) GetElasticCommonData() any { return nil }

resources/fetchers/file_system_fetcher.go

@@ -20,14 +20,16 @@ package fetchers
 import (
 	"context"
 	"fmt"
-	"os"
-	"strconv"
-	"syscall"
-
+	"github.com/djherbis/times"
 	"github.com/elastic/cloudbeat/resources/fetching"
 	"github.com/elastic/cloudbeat/resources/utils/user"
-
+	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+	"time"
 
 	"github.com/elastic/elastic-agent-libs/logp"
 )
@@ -40,18 +42,42 @@ const (
 	GroupFile      = "/hostfs/etc/group"
 )
 
-type FileSystemResource struct {
+type EvalFSResource struct {
 	Name    string `json:"name"`
 	Mode    string `json:"mode"`
-	Gid     uint32 `json:"gid"`
-	Uid     uint32 `json:"uid"`
+	Gid     string `json:"gid"`
+	Uid     string `json:"uid"`
 	Owner   string `json:"owner"`
 	Group   string `json:"group"`
 	Path    string `json:"path"`
 	Inode   string `json:"inode"`
 	SubType string `json:"sub_type"`
 }
 
+// FileCommonData According to https://www.elastic.co/guide/en/ecs/current/ecs-file.html
+type FileCommonData struct {
+	Name      string    `json:"name,omitempty"`
+	Mode      string    `json:"mode,omitempty"`
+	Gid       string    `json:"gid,omitempty"`
+	Uid       string    `json:"uid,omitempty"`
+	Owner     string    `json:"owner,omitempty"`
+	Group     string    `json:"group,omitempty"`
+	Path      string    `json:"path,omitempty"`
+	Inode     string    `json:"inode,omitempty"`
+	Extension string    `json:"extension,omitempty"`
+	Size      int64     `json:"size"`
+	Type      string    `json:"type,omitempty"`
+	Directory string    `json:"directory,omitempty"`
+	Accessed  time.Time `json:"accessed"`
+	Mtime     time.Time `json:"mtime"`
+	Ctime     time.Time `json:"ctime"`
+}
+
+type FSResource struct {
+	EvalResource  EvalFSResource
+	ElasticCommon FileCommonData
+}
+
 // FileSystemFetcher implement the Fetcher interface
 // The FileSystemFetcher meant to fetch file/directories from the file system and ship it
 // to the Cloudbeat
@@ -91,44 +117,44 @@ func (f *FileSystemFetcher) Fetch(ctx context.Context, cMetadata fetching.CycleM
 	return nil
 }
 
-func (f *FileSystemFetcher) fetchSystemResource(filePath string) (FileSystemResource, error) {
+func (f *FileSystemFetcher) fetchSystemResource(filePath string) (FSResource, error) {
 
 	info, err := os.Stat(filePath)
 	if err != nil {
-		return FileSystemResource{}, fmt.Errorf("failed to fetch %s, error: %w", filePath, err)
+		return FSResource{}, fmt.Errorf("failed to fetch %s, error: %w", filePath, err)
 	}
 	resourceInfo, _ := f.fromFileInfo(info, filePath)
 
 	return resourceInfo, nil
 }
 
-func (f *FileSystemFetcher) fromFileInfo(info os.FileInfo, path string) (FileSystemResource, error) {
+func (f *FileSystemFetcher) fromFileInfo(info os.FileInfo, path string) (FSResource, error) {
 
 	if info == nil {
-		return FileSystemResource{}, nil
+		return FSResource{}, nil
 	}
 
 	stat, ok := info.Sys().(*syscall.Stat_t)
 	if !ok {
-		return FileSystemResource{}, errors.New("Not a syscall.Stat_t")
+		return FSResource{}, errors.New("Not a syscall.Stat_t")
 	}
 
 	mod := strconv.FormatUint(uint64(info.Mode().Perm()), 8)
+	uid := strconv.FormatUint(uint64(stat.Uid), 10)
+	gid := strconv.FormatUint(uint64(stat.Gid), 10)
 	inode := strconv.FormatUint(stat.Ino, 10)
 
-	uid := stat.Uid
-	gid := stat.Gid
 	username, err := f.osUser.GetUserNameFromID(uid, UserFile)
 	if err != nil {
-		logp.Error(fmt.Errorf("failed to find username for uid %d, error - %+v", uid, err))
+		logp.Error(fmt.Errorf("failed to find username for uid %s, error - %+v", uid, err))
 	}
 
 	groupName, err := f.osUser.GetGroupNameFromID(gid, GroupFile)
 	if err != nil {
-		logp.Error(fmt.Errorf("failed to find groupname for gid %d, error - %+v", gid, err))
+		logp.Error(fmt.Errorf("failed to find groupname for gid %s, error - %+v", gid, err))
 	}
 
-	data := FileSystemResource{
+	data := EvalFSResource{
 		Name:    info.Name(),
 		Mode:    mod,
 		Gid:     gid,
@@ -140,22 +166,30 @@ func (f *FileSystemFetcher) fromFileInfo(info os.FileInfo, path string) (FileSys
 		SubType: getFSSubType(info),
 	}
 
-	return data, nil
+	return FSResource{
+		EvalResource:  data,
+		ElasticCommon: enrichFileCommonData(stat, data, path),
+	}, nil
 }
 
 func (f *FileSystemFetcher) Stop() {
 }
 
-func (r FileSystemResource) GetData() interface{} {
-	return r
+func (r FSResource) GetData() any {
+	return r.EvalResource
 }
 
-func (r FileSystemResource) GetMetadata() fetching.ResourceMetadata {
+func (r FSResource) GetElasticCommonData() any {
+	return r.ElasticCommon
+}
+
+func (r FSResource) GetMetadata() fetching.ResourceMetadata {
 	return fetching.ResourceMetadata{
-		ID:      r.Path,
-		Type:    FSResourceType,
-		SubType: r.SubType,
-		Name:    r.Path, // The Path from the container and not from the host
+		ID:        r.EvalResource.Path,
+		Type:      FSResourceType,
+		SubType:   r.EvalResource.SubType,
+		Name:      r.EvalResource.Path, // The Path from the container and not from the host
+		ECSFormat: FSResourceType,
 	}
 }
 
@@ -165,3 +199,41 @@ func getFSSubType(fileInfo os.FileInfo) string {
 	}
 	return FileSubType
 }
+
+func enrichFileCommonData(stat *syscall.Stat_t, data EvalFSResource, path string) FileCommonData {
+	cd := FileCommonData{}
+	if err := enrichFromFileResource(&cd, data); err != nil {
+		logp.Error(fmt.Errorf("failed to decode data, Error: %v", err))
+	}
+
+	if err := enrichFileTimes(&cd, path); err != nil {
+		logp.Error(err)
+	}
+
+	cd.Extension = filepath.Ext(path)
+	cd.Directory = filepath.Dir(path)
+	cd.Size = stat.Size
+	cd.Type = data.SubType
+
+	return cd
+}
+
+func enrichFileTimes(cd *FileCommonData, filepath string) error {
+	t, err := times.Stat(filepath)
+	if err != nil {
+		return fmt.Errorf("failed to get file time data, error - %+v", err)
+	}
+
+	cd.Accessed = t.AccessTime()
+	cd.Mtime = t.ModTime()
+
+	if t.HasChangeTime() {
+		cd.Ctime = t.ChangeTime()
+	}
+
+	return nil
+}
+
+func enrichFromFileResource(cd *FileCommonData, data EvalFSResource) error {
+	return mapstructure.Decode(data, cd)
+}