diff --git a/README.md b/README.md index 48c16020..1600e362 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ - [Usage](#usage) - [Configuration](#configuration) - [Startup](#startup) + - [Loader](#loader) - [Runtime](#runtime) - [Check: Health](#check-health) - [API](#api) @@ -71,6 +72,18 @@ Priority of configuration (high to low): 3. Defined configuration file 4. Default configuration file +#### Loader + +The loader component of the `sparrow` will load the [Runtime](#runtime) configuration dynamically. + +The loader can be selected by specifying the `loaderType` configuration parameter. + +The default loader is an `http` loader that is able to get the runtime configuration from a remote endpoint. + +Available loader: +- `http`: The default. Loads configuration from a remote endpoint. Token authentication is available. Additional configuration parameter have the prefix `loaderHttp`. +- `file` (experimental): Loads configuration once from a local file. Additional configuration parameter have the prefix `loaderFile`. This is just for development purposes. + ### Runtime Besides the technical startup configuration the configuration for the `sparrow` checks is loaded dynamically from an http endpoint. The `loader` is able to load the configuration dynamically during the runtime. Checks can be enabled, disabled and configured. The available loader confutation options for the startup configuration can be found in [here](sparrow_run.md) diff --git a/cmd/run.go b/cmd/run.go index f93ce0b3..ef6b876c 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -3,11 +3,12 @@ package cmd import ( "context" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/caas-team/sparrow/internal/logger" "github.com/caas-team/sparrow/pkg/config" "github.com/caas-team/sparrow/pkg/sparrow" - "github.com/spf13/cobra" - "github.com/spf13/viper" ) // NewCmdRun creates a new run command @@ -21,6 +22,7 @@ func NewCmdRun() *cobra.Command { LoaderHttpTimeout: "loaderHttpTimeout", LoaderHttpRetryCount: "loaderHttpRetryCount", LoaderHttpRetryDelay: "loaderHttpRetryDelay", + LoaderFilePath: "loaderFilePath", } cmd := &cobra.Command{ @@ -32,13 +34,15 @@ func NewCmdRun() *cobra.Command { cmd.PersistentFlags().String(flagMapping.ApiListeningAddress, ":8080", "api: The address the server is listening on") - cmd.PersistentFlags().StringP(flagMapping.LoaderType, "l", "http", "defines the loader type that will load the checks configuration during the runtime") + cmd.PersistentFlags().StringP(flagMapping.LoaderType, "l", "http", + "defines the loader type that will load the checks configuration during the runtime. The fallback is the fileLoader") cmd.PersistentFlags().Int(flagMapping.LoaderInterval, 300, "defines the interval the loader reloads the configuration in seconds") cmd.PersistentFlags().String(flagMapping.LoaderHttpUrl, "", "http loader: The url where to get the remote configuration") cmd.PersistentFlags().String(flagMapping.LoaderHttpToken, "", "http loader: Bearer token to authenticate the http endpoint") cmd.PersistentFlags().Int(flagMapping.LoaderHttpTimeout, 30, "http loader: The timeout for the http request in seconds") cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryCount, 3, "http loader: Amount of retries trying to load the configuration") cmd.PersistentFlags().Int(flagMapping.LoaderHttpRetryDelay, 1, "http loader: The initial delay between retries in seconds") + cmd.PersistentFlags().String(flagMapping.LoaderFilePath, "config.yaml", "file loader: The path to the file to read the runtime config from") viper.BindPFlag(flagMapping.ApiListeningAddress, cmd.PersistentFlags().Lookup(flagMapping.ApiListeningAddress)) @@ -49,6 +53,7 @@ func NewCmdRun() *cobra.Command { viper.BindPFlag(flagMapping.LoaderHttpTimeout, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpTimeout)) viper.BindPFlag(flagMapping.LoaderHttpRetryCount, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryCount)) viper.BindPFlag(flagMapping.LoaderHttpRetryDelay, cmd.PersistentFlags().Lookup(flagMapping.LoaderHttpRetryDelay)) + viper.BindPFlag(flagMapping.LoaderFilePath, cmd.PersistentFlags().Lookup(flagMapping.LoaderFilePath)) return cmd } @@ -70,6 +75,7 @@ func run(fm *config.RunFlagsNameMapping) func(cmd *cobra.Command, args []string) cfg.SetLoaderHttpTimeout(viper.GetInt(fm.LoaderHttpTimeout)) cfg.SetLoaderHttpRetryCount(viper.GetInt(fm.LoaderHttpRetryCount)) cfg.SetLoaderHttpRetryDelay(viper.GetInt(fm.LoaderHttpRetryDelay)) + cfg.SetLoaderFilePath(viper.GetString(fm.LoaderFilePath)) if err := cfg.Validate(ctx, fm); err != nil { log.Error("Error while validating the config", "error", err) diff --git a/pkg/checks/checks.go b/pkg/checks/checks.go index 955cae55..a4cbbb16 100644 --- a/pkg/checks/checks.go +++ b/pkg/checks/checks.go @@ -4,16 +4,16 @@ import ( "context" "time" - "github.com/caas-team/sparrow/pkg/api" "github.com/getkin/kin-openapi/openapi3" + + "github.com/caas-team/sparrow/pkg/api" ) // Available Checks will be registered in this map // The key is the name of the Check // The name needs to map the configuration item key var RegisteredChecks = map[string]func() Check{ - "rtt": GetRoundtripCheck, - "health": GetHealthCheck, + "health": NewHealthCheck, } //go:generate moq -out checks_moq.go . Check diff --git a/pkg/checks/health.go b/pkg/checks/health.go index 4dc17ada..b76cafdd 100644 --- a/pkg/checks/health.go +++ b/pkg/checks/health.go @@ -40,7 +40,7 @@ type Target struct { } // Constructor for the HealthCheck -func GetHealthCheck() Check { +func NewHealthCheck() Check { return &Health{ route: "health", } diff --git a/pkg/checks/roundtrip.go b/pkg/checks/roundtrip.go deleted file mode 100644 index 92f0d0bc..00000000 --- a/pkg/checks/roundtrip.go +++ /dev/null @@ -1,90 +0,0 @@ -package checks - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/caas-team/sparrow/internal/logger" - "github.com/caas-team/sparrow/pkg/api" - "github.com/getkin/kin-openapi/openapi3" -) - -// ensure that RoundTrip implements the Check interface -var _ Check = (*RoundTrip)(nil) - -type RoundTripConfig struct{} -type roundTripData struct { - Ms int64 `json:"ms"` -} - -// RoundTrip is a check that measures the round trip time of a request -type RoundTrip struct { - c chan<- Result - config RoundTripConfig -} - -// Constructor for the RoundtripCheck -func GetRoundtripCheck() Check { - return &RoundTrip{} -} - -func (rt *RoundTrip) Run(ctx context.Context) error { - ctx, cancel := logger.NewContextWithLogger(ctx, "roundTrip") - defer cancel() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Second): - fmt.Println("Sending data to db") - rt.c <- Result{Timestamp: time.Now(), Err: "", Data: roundTripData{Ms: 1000}} - } - - } -} - -func (rt *RoundTrip) Startup(ctx context.Context, cResult chan<- Result) error { - // TODO register http handler for this check - http.HandleFunc("rtt", func(w http.ResponseWriter, r *http.Request) { - // TODO handle - }) - - rt.c = cResult - return nil -} - -// Shutdown is called once when the check is unregistered or sparrow shuts down - -func (rt *RoundTrip) Shutdown(ctx context.Context) error { - http.Handle("rtt", http.NotFoundHandler()) - - return nil -} - -func (rt *RoundTrip) SetConfig(ctx context.Context, config any) error { - checkConfig, ok := config.(RoundTripConfig) - if !ok { - return ErrInvalidConfig - } - rt.config = checkConfig - return nil -} - -func (rt *RoundTrip) Schema() (*openapi3.SchemaRef, error) { - return OpenapiFromPerfData[roundTripData](roundTripData{}) - -} - -func (rt *RoundTrip) RegisterHandler(ctx context.Context, router *api.RoutingTree) { - router.Add(http.MethodGet, "rtt", rt.handleRoundtrip) -} - -func (rt *RoundTrip) DeregisterHandler(ctx context.Context, router *api.RoutingTree) { - router.Remove(http.MethodGet, "rtt") -} - -func (rt *RoundTrip) handleRoundtrip(w http.ResponseWriter, r *http.Request) { - // TODO handle -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 2225f629..ec542685 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,7 @@ type LoaderConfig struct { Type string Interval time.Duration http HttpLoaderConfig + file FileLoaderConfig } // HttpLoaderConfig is the configuration @@ -33,6 +34,10 @@ type HttpLoaderConfig struct { retryCfg helper.RetryConfig } +type FileLoaderConfig struct { + path string +} + // NewConfig creates a new Config func NewConfig() *Config { return &Config{ @@ -49,6 +54,10 @@ func (c *Config) SetLoaderType(loaderType string) { c.Loader.Type = loaderType } +func (c *Config) SetLoaderFilePath(loaderFilePath string) { + c.Loader.file.path = loaderFilePath +} + // SetLoaderInterval sets the loader interval // loaderInterval in seconds func (c *Config) SetLoaderInterval(loaderInterval int) { diff --git a/pkg/config/file.go b/pkg/config/file.go new file mode 100644 index 00000000..296358f1 --- /dev/null +++ b/pkg/config/file.go @@ -0,0 +1,44 @@ +package config + +import ( + "context" + "os" + + "gopkg.in/yaml.v3" + + "github.com/caas-team/sparrow/internal/logger" +) + +var _ Loader = (*FileLoader)(nil) + +type FileLoader struct { + path string + c chan<- map[string]any +} + +func NewFileLoader(cfg *Config, cCfgChecks chan<- map[string]any) *FileLoader { + return &FileLoader{ + path: cfg.Loader.file.path, + c: cCfgChecks, + } +} + +func (f *FileLoader) Run(ctx context.Context) { + log := logger.FromContext(ctx).WithGroup("FileLoader") + log.Info("Reading config from file", "file", f.path) + // TODO refactor this to use fs.FS + b, err := os.ReadFile(f.path) + if err != nil { + log.Error("Failed to read config file", "path", f.path, "error", err) + panic("failed to read config file " + err.Error()) + } + + var cfg RuntimeConfig + + if err := yaml.Unmarshal(b, &cfg); err != nil { + log.Error("Failed to parse config file", "error", err) + panic("failed to parse config file: " + err.Error()) + } + + f.c <- cfg.Checks +} diff --git a/pkg/config/file_test.go b/pkg/config/file_test.go new file mode 100644 index 00000000..2c6b9ec6 --- /dev/null +++ b/pkg/config/file_test.go @@ -0,0 +1,59 @@ +package config + +import ( + "context" + "reflect" + "testing" +) + +func TestNewFileLoader(t *testing.T) { + l := NewFileLoader(&Config{Loader: LoaderConfig{file: FileLoaderConfig{path: "config.yaml"}}}, make(chan<- map[string]any)) + + if l.path != "config.yaml" { + t.Errorf("Expected path to be config.yaml, got %s", l.path) + } + if l.c == nil { + t.Errorf("Expected channel to be not nil") + } +} + +func TestFileLoader_Run(t *testing.T) { + type fields struct { + path string + c chan map[string]any + } + type args struct { + ctx *context.Context + cancel *context.CancelFunc + } + type want struct { + cfg map[string]any + } + tests := []struct { + name string + fields fields + args args + want want + }{ + {name: "Loads config from file", fields: fields{path: "testdata/config.yaml", c: make(chan map[string]any)}, args: func() args { + ctx, cancel := context.WithCancel(context.Background()) + return args{ctx: &ctx, cancel: &cancel} + }(), want: want{cfg: map[string]any{"testCheck1": map[string]any{"enabled": true}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &FileLoader{ + path: tt.fields.path, + c: tt.fields.c, + } + go f.Run(*tt.args.ctx) + (*tt.args.cancel)() + + config := <-tt.fields.c + + if !reflect.DeepEqual(config, tt.want.cfg) { + t.Errorf("Expected config to be %v, got %v", tt.want.cfg, config) + } + }) + } +} diff --git a/pkg/config/flags.go b/pkg/config/flags.go index 6a08a499..1825016e 100644 --- a/pkg/config/flags.go +++ b/pkg/config/flags.go @@ -10,4 +10,5 @@ type RunFlagsNameMapping struct { LoaderHttpTimeout string LoaderHttpRetryCount string LoaderHttpRetryDelay string + LoaderFilePath string } diff --git a/pkg/config/loader.go b/pkg/config/loader.go index 67bced3a..2e04f469 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -15,5 +15,10 @@ type Loader interface { // Get a new typed runtime configuration loader func NewLoader(cfg *Config, cCfgChecks chan<- map[string]any) Loader { - return NewHttpLoader(cfg, cCfgChecks) + switch cfg.Loader.Type { + case "http": + return NewHttpLoader(cfg, cCfgChecks) + default: + return NewFileLoader(cfg, cCfgChecks) + } } diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 8d37d95d..fcd05510 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -15,16 +15,18 @@ func (c *Config) Validate(ctx context.Context, fm *RunFlagsNameMapping) error { log := logger.FromContext(ctx) ok := true - if _, err := url.ParseRequestURI(c.Loader.http.url); err != nil { - ok = false - log.ErrorContext(ctx, "The loader http url is not a valid url", - fm.LoaderHttpUrl, c.Loader.http.url) - } - - if c.Loader.http.retryCfg.Count < 0 || c.Loader.http.retryCfg.Count >= 5 { - ok = false - log.Error("The amount of loader http retries should be above 0 and below 6", - fm.LoaderHttpRetryCount, c.Loader.http.retryCfg.Count) + switch c.Loader.Type { + case "http": + if _, err := url.ParseRequestURI(c.Loader.http.url); err != nil { + ok = false + log.ErrorContext(ctx, "The loader http url is not a valid url", + fm.LoaderHttpUrl, c.Loader.http.url) + } + if c.Loader.http.retryCfg.Count < 0 || c.Loader.http.retryCfg.Count >= 5 { + ok = false + log.Error("The amount of loader http retries should be above 0 and below 6", + fm.LoaderHttpRetryCount, c.Loader.http.retryCfg.Count) + } } if !ok { diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go index cb25e270..466132d2 100644 --- a/pkg/config/validate_test.go +++ b/pkg/config/validate_test.go @@ -25,6 +25,7 @@ func TestConfig_Validate(t *testing.T) { name: "config ok", fields: fields{ Loader: LoaderConfig{ + Type: "http", http: HttpLoaderConfig{ url: "https://test.de/config", timeout: time.Second, @@ -42,6 +43,7 @@ func TestConfig_Validate(t *testing.T) { name: "url missing", fields: fields{ Loader: LoaderConfig{ + Type: "http", http: HttpLoaderConfig{ url: "", timeout: time.Second, @@ -59,6 +61,7 @@ func TestConfig_Validate(t *testing.T) { name: "url malformed", fields: fields{ Loader: LoaderConfig{ + Type: "http", http: HttpLoaderConfig{ url: "this is not a valid url", timeout: time.Second, @@ -76,6 +79,7 @@ func TestConfig_Validate(t *testing.T) { name: "retry count to high", fields: fields{ Loader: LoaderConfig{ + Type: "http", http: HttpLoaderConfig{ url: "test.de", timeout: time.Minute, diff --git a/pkg/sparrow/run_test.go b/pkg/sparrow/run_test.go index 97fe4330..7dc173d8 100644 --- a/pkg/sparrow/run_test.go +++ b/pkg/sparrow/run_test.go @@ -7,13 +7,14 @@ import ( "testing" "time" + "github.com/getkin/kin-openapi/openapi3" + "github.com/stretchr/testify/assert" + "github.com/caas-team/sparrow/internal/logger" "github.com/caas-team/sparrow/pkg/api" "github.com/caas-team/sparrow/pkg/checks" "github.com/caas-team/sparrow/pkg/config" "github.com/caas-team/sparrow/pkg/db" - "github.com/getkin/kin-openapi/openapi3" - "github.com/stretchr/testify/assert" ) func TestSparrow_ReconcileChecks(t *testing.T) { @@ -144,7 +145,6 @@ func TestSparrow_ReconcileChecks(t *testing.T) { } func Test_fanInResults(t *testing.T) { - checkChan := make(chan checks.Result) cResult := make(chan checks.ResultDTO) name := "check" @@ -174,6 +174,9 @@ func Test_fanInResults(t *testing.T) { func TestSparrow_Run(t *testing.T) { c := &config.Config{ Api: config.ApiConfig{ListeningAddress: ":9090"}, + Loader: config.LoaderConfig{ + Type: "http", + }, } s := New(c)