Move auto reload and auto update resources. (#8)

* remove app reloader from bundler package * add tests for file copy helpers * check content when copying files * missing migrations test file * update dependencies * make the conf prefix configurable * fix: use provided bundle dir for upated assets in appreloader * chore: Updated coverage badge. --------- Co-authored-by: GitHub Action <[email protected]>
dhontecillas · Nov 12, 2023 · 9008a76 · 9008a76
1 parent a59fb2d
commit 9008a76
Show file tree

Hide file tree

Showing 13 changed files with 271 additions and 154 deletions.
diff --git a/Makefile b/Makefile
@@ -11,7 +11,7 @@ GOTOOLS          ?= github.com/GeertJohan/fgt \
 
 
 build:
-	go build -v ./cmd/importer
+	go build -v ./cmd/collectmigrations
 .PHONY: build
 
 lint: tools

diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 # **HFW**: Handy FrameWork
-![Coverage](https://img.shields.io/badge/Coverage-26.9%25-red)
+![Coverage](https://img.shields.io/badge/Coverage-31.4%25-yellow)
 
 [![Coverage Status](https://coveralls.io/repos/github/dhontecillas/hfw/badge.svg?branch=main)](https://coveralls.io/github/dhontecillas/hfw?branch=main)
 
@@ -277,3 +277,9 @@ These are the stable APIs for mobile apps or third party integrations.
 - Errors are logged at the usecase layer whenever is possible.
 - Errors coming from other libraries that aren't know
 
+
+# Alternatives 
+
+These are much more elaborated frameworks, production ready that you might want to have a look at:
+
+- [Iris](https://github.com/kataras/iris)
diff --git a/cmd/bundler/main.go b/cmd/bundler/main.go
@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/viper"
+
+	"github.com/dhontecillas/hfw/pkg/bundler"
+	"github.com/dhontecillas/hfw/pkg/config"
+	"github.com/dhontecillas/hfw/pkg/obs/metrics"
+)
+
+const (
+	EnvKeyBundlerConfPrefix string = "HFW_BUNDLER_PREFIX"
+)
+
+func main() {
+	confPrefix := os.Getenv(EnvKeyBundlerConfPrefix)
+	if len(confPrefix) == 0 {
+		fmt.Printf("the HFW_BUNDLER_PREFIX must be set and should be something like 'yourappprefix.'\n")
+		return
+	}
+
+	if err := config.InitConfig(confPrefix); err != nil {
+		panic(err)
+	}
+
+	insConf := config.ReadInsightsConfig(confPrefix)
+	if insConf == nil {
+		panic("insConf is null")
+	}
+	appMetricDefs := make(metrics.Defs, 0)
+	insB, insF := config.CreateInsightsBuilder(insConf, appMetricDefs)
+	ins := insB()
+	defer insF()
+
+	bundler.ExecuteBundlerOperations(viper.GetViper(), ins.L, confPrefix)
+}
diff --git a/docs/bundler.md b/docs/bundler.md
@@ -1,13 +1,13 @@
 # **HFW***'s Bundler
 
-The `bundler` package identifies certain paths as containing static
-assets:
+The `bundler` package identifies certain paths as containing static assets:
 
 - `*/static` : files that should be served as they are, without going through the app
 - `*/html_templates` : those are files that should be stored in the server, in an
     accessible place for the app to load them and use them to render the ouptut
 - `*/notifications/templates` : those files are collected, and used to send notifications
-
+- `*/migrations` : database migration files with a name matching the format: `[number]_[descriptive_name].[up|down].sql` that are used by the `github.com/golang-migrate/migrate`
+- `/config/*.yaml` : config files that can 
 
 ## The "bundling" process
 
@@ -17,22 +17,18 @@ deployed to a server.
 It consists on the following directories:
 
 - `app` : The place where the app executable will be put, with the files that the
-    app must be able to acces:
-    - `app/dbmigrations` : the collected migration files, that can be run before
+    app must be able to access:
+    - `app/dbmigrations`: the collected migration files, that can be run before
         start the application (be careful when launching several instances of
         the same app, as concurrency has not been tested: in that case it might
         be better to run migrations as a separate initial process when doing a
         new deployment).
-    - `app/data`: tempaltes to be used by the app
+    - `app/data`: templates to be used by the app
         - `app/data/html_templates`: html templates to render
         - `app/data/notifications/templates`: templates to be used for messages
+    - `app/config`: config files to be used at the startup of the 
 - `static` : A directory containing all the collected static files, in a single place, so we can put them in a place to be directly served by an http server, or some CDN
 
-### Gathering files
-
-Be careful of not storing the bundle under a subdir that is in the path of the files
-to collect or you would end with a recursive file structure.
-
 
 ### Compiling the executable
 

diff --git a/go.mod b/go.mod
@@ -1,149 +1,87 @@
 module github.com/dhontecillas/hfw
 
-go 1.18
+go 1.21.3
 
 require (
-	github.com/DataDog/datadog-go v4.8.3+incompatible
-	github.com/getsentry/sentry-go v0.13.0
+	github.com/DataDog/datadog-go/v5 v5.3.0
+	github.com/getsentry/sentry-go v0.25.0
 	github.com/gin-contrib/cors v1.4.0
-	github.com/gin-contrib/multitemplate v0.0.0-20220705015713-e21a0ba39de3
+	github.com/gin-contrib/multitemplate v0.0.0-20230212012517-45920c92c271
 	github.com/gin-contrib/sessions v0.0.5
-	github.com/gin-gonic/gin v1.8.1
-	github.com/golang-migrate/migrate/v4 v4.15.2
+	github.com/gin-gonic/gin v1.9.1
+	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/jmoiron/sqlx v1.3.5
-	github.com/lib/pq v1.10.6
-	github.com/mailgun/mailgun-go/v4 v4.8.1
+	github.com/lib/pq v1.10.9
+	github.com/mailgun/mailgun-go/v4 v4.11.0
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.13.0
+	github.com/prometheus/client_golang v1.17.0
 	github.com/radovskyb/watcher v1.0.7
-	github.com/sendgrid/sendgrid-go v3.11.1+incompatible
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/viper v1.12.0
+	github.com/sendgrid/sendgrid-go v3.13.0+incompatible
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/viper v1.17.0
 	github.com/utrack/gin-csrf v0.0.0-20190424104817-40fb8d2c8fca
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
+	golang.org/x/crypto v0.14.0
 	gopkg.in/gemnasium/logrus-graylog-hook.v2 v2.0.7
 )
 
 require (
-	github.com/BurntSushi/toml v1.2.1 // indirect
-	github.com/Djarvur/go-err113 v0.0.0-20200410182137-af658d038157 // indirect
-	github.com/GeertJohan/fgt v0.0.0-20160120143236-262f7b11eec0 // indirect
-	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
-	github.com/bombsimon/wsl/v3 v3.0.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 // indirect
-	github.com/fatih/color v1.13.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
+	github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-critic/go-critic v0.4.1 // indirect
-	github.com/go-lintpack/lintpack v0.5.2 // indirect
-	github.com/go-playground/locales v0.14.0 // indirect
-	github.com/go-playground/universal-translator v0.18.0 // indirect
-	github.com/go-playground/validator/v10 v10.11.0 // indirect
-	github.com/go-toolsmith/astcast v1.0.0 // indirect
-	github.com/go-toolsmith/astcopy v1.0.0 // indirect
-	github.com/go-toolsmith/astequal v1.0.0 // indirect
-	github.com/go-toolsmith/astfmt v1.0.0 // indirect
-	github.com/go-toolsmith/astp v1.0.0 // indirect
-	github.com/go-toolsmith/strparse v1.0.0 // indirect
-	github.com/go-toolsmith/typep v1.0.0 // indirect
-	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/goccy/go-json v0.9.10 // indirect
-	github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
-	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
-	github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee // indirect
-	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
-	github.com/golangci/golangci-lint v1.27.0 // indirect
-	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc // indirect
-	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
-	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
-	github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 // indirect
-	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
-	github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 // indirect
-	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/go-chi/chi/v5 v5.0.8 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/gorilla/context v1.1.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
 	github.com/gorilla/sessions v1.2.1 // indirect
-	github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a // indirect
-	github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/kisielk/errcheck v1.6.3 // indirect
-	github.com/kisielk/gotool v1.0.0 // indirect
-	github.com/leodido/go-urn v1.2.1 // indirect
-	github.com/magiconair/properties v1.8.6 // indirect
-	github.com/maratori/testpackage v1.0.1 // indirect
-	github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/goveralls v0.0.11 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/nakabonne/nestif v0.3.0 // indirect
-	github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
-	github.com/pelletier/go-toml v1.9.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/ryancurrah/gomodguard v1.0.4 // indirect
-	github.com/securego/gosec/v2 v2.3.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sendgrid/rest v2.6.9+incompatible // indirect
-	github.com/sourcegraph/go-diff v0.5.1 // indirect
-	github.com/spf13/afero v1.9.2 // indirect
-	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/cobra v1.1.3 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.4.0 // indirect
-	github.com/stretchr/testify v1.8.0 // indirect
-	github.com/subosito/gotenv v1.4.0 // indirect
-	github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 // indirect
-	github.com/tetafro/godot v0.3.7 // indirect
-	github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e // indirect
-	github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa // indirect
-	github.com/ugorji/go/codec v1.2.7 // indirect
-	github.com/ultraware/funlen v0.0.2 // indirect
-	github.com/ultraware/whitespace v0.0.4 // indirect
-	github.com/uudashr/gocognit v1.0.1 // indirect
-	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.6.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.11 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/arch v0.3.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	honnef.co/go/tools v0.4.1 // indirect
-	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
-	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
-	mvdan.cc/unparam v0.0.0-20190720180237-d51796306d8f // indirect
-	sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 // indirect
 )
diff --git a/pkg/bundler/app_updater.go → pkg/appreload/appreloader.go b/pkg/bundler/app_updater.go → pkg/appreload/appreloader.go
@@ -1,4 +1,8 @@
-package bundler
+// Package appreload allows to detect changes in the source code for
+// a Go application and rebuild the executable. In case of success it
+// can kill the previous running version, and launch the newly created
+// executable.
+package appreload
 
 import (
 	"fmt"
@@ -10,15 +14,16 @@ import (
 
 	"github.com/radovskyb/watcher"
 
+	"github.com/dhontecillas/hfw/pkg/bundler"
 	"github.com/dhontecillas/hfw/pkg/obs"
 )
 
 // AppUpdaterConf has the config vars to set up an AppUpdater instance:
-// - ExecDir: the main sources directory for the executable (is also used
-// 		to generate the temp binary file name)
-// - PkgsDirs: the list of directories to observe for changes
-// - BuildFileExts: file extensions that should trigger a new binary build
-// - ResFilesExts: file extensions that should trigger an update of the resource files
+//   - ExecDir: the main sources directory for the executable (is also used
+//     to generate the temp binary file name)
+//   - PkgsDirs: the list of directories to observe for changes
+//   - BuildFileExts: file extensions that should trigger a new binary build
+//   - ResFilesExts: file extensions that should trigger an update of the resource files
 type AppUpdaterConf struct {
 	ExecDir       string
 	PkgsDirs      []string
@@ -59,7 +64,7 @@ func (a *AppUpdater) Launch() {
 	for _, pkgDir := range a.conf.PkgsDirs {
 		if err := a.w.AddRecursive(pkgDir); err != nil {
 			a.ins.L.Err(err, fmt.Sprintf("cannot add recursive dir %s : %s",
-				a.conf.ExecDir, err.Error()))
+				pkgDir, err.Error()))
 		}
 	}
 
@@ -113,6 +118,7 @@ func (a *AppUpdater) procRegularFile(event *watcher.Event) {
 
 	for _, ext := range a.conf.ResFileExts {
 		if strings.HasSuffix(nm, ext) {
+			a.ins.L.InfoMsg("updating resource").Str("resource", nm).Send()
 			a.UpdateResource(event)
 			return
 		}
@@ -189,19 +195,19 @@ func (a *AppUpdater) UpdateResource(e *watcher.Event) {
 	if !e.Mode().IsRegular() {
 		return
 	}
-	dataDirs := DataDirs()
+	dataDirs := bundler.DataDirs()
 
 	a.ins.L.Warn(fmt.Sprintf("update %s", e.Path))
 	for dst, dir := range dataDirs {
 		if idx := strings.Index(e.Path, dir); idx >= 0 {
-			dstPath := filepath.Join("./bundle", dst, e.Path[idx+len(dir):])
+			dstPath := filepath.Join(a.conf.BundleDir, dst, e.Path[idx+len(dir):])
 			parentDir := filepath.Dir(dstPath)
 			if err := os.MkdirAll(parentDir, 0775); err != nil {
 				a.ins.L.Err(err, fmt.Sprintf("error creating parent dir %s: %s",
 					parentDir, err.Error()))
 				continue
 			}
-			if err := CopyFile(e.Path, dstPath); err != nil {
+			if err := bundler.CopyFile(e.Path, dstPath); err != nil {
 				a.ins.L.Err(err, fmt.Sprintf("error copying file %s to %s: %s",
 					e.Path, dstPath, err.Error()))
 			}