Pull request #6: Mr/aj review

Merge in AW/integresql from mr/aj-review to aj/pooling-improvements

* commit '7b99232ec350445edf06f1b85dc650f32b32616d':
  don't lock while extending, reuse recreateDatabaseGracefully
  intro zerolog and add proper log statements in pool and manager, into typical go-starter logging middleware and additional env vars for controlling them
  private disableWorkerAutostart
  fix toolchain, reenable pipefails, fix linting, cleanup
  adds test database generations and TestDatabaseMinimalLifetime (blocks auto clean for specific time on issued databases) to properly deal with pressure caused by fast test db issuance and out-of-order recreates which might cause interferances
  adds backoff handling for recreateDatabaseGracefully
  redesign recreate handling
  filter via tmp channel without closing the initial, fix tests / stabilize via disabling worker autostart
  close, range and refill dirty channel on unlock and recreate, debug statements, use 1000 instead of 10e3
  review only, reactivate print debugging

.devcontainer/devcontainer.json

@@ -14,9 +14,11 @@
 	// The optional 'workspaceFolder' property is the path VS Code should open by default when
 	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
 	"workspaceFolder": "/app",
-	// Set *default* container specific settings.json values on container create.
+	// All containers should stop if we close / reload the VSCode window.
+	"shutdownAction": "stopCompose",
 	"customizations": {
 		"vscode": {
+			// Set *default* container specific settings.json values on container create.
 			"settings": {
 				// https://github.com/golang/tools/blob/master/gopls/doc/vscode.md#vscode
 				"go.useLanguageServer": true,
@@ -45,6 +47,8 @@
 					// DISABLED, done via
 					"staticcheck": false,
 				},
+				// https://code.visualstudio.com/docs/languages/go#_intellisense
+				"go.autocompleteUnimportedPackages": true,
 				// https://github.com/golangci/golangci-lint#editor-integration
 				"go.lintTool": "golangci-lint",
 				"go.lintFlags": [
@@ -69,43 +73,19 @@
 				},
 				// ensure that the pgFormatter VSCode extension uses the pgFormatter that comes preinstalled in the Dockerfile
 				"pgFormatter.pgFormatterPath": "/usr/local/bin/pg_format"
-				// "go.lintOnSave": "workspace"
-				// general build settings in sync with our makefile
-				// "go.buildFlags": [
-				// 	"-o",
-				// 	"bin/app"
-				// ]
-				// "sqltools.connections": [
-				// 	{
-				// 		"database": "sample",
-				// 		"dialect": "PostgreSQL",
-				// 		"name": "postgres",
-				// 		"password": "9bed16f749d74a3c8bfbced18a7647f5",
-				// 		"port": 5432,
-				// 		"server": "postgres",
-				// 		"username": "dbuser"
-				// 	}
-				// ],
-				// "sqltools.autoConnectTo": [
-				// 	"postgres"
-				// ],
-				// // only use pg_format to actually format!
-				// "sqltools.formatLanguages": [],
-				// "sqltools.telemetry": false,
-				// "sqltools.autoOpenSessionFiles": false
 			},
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				// required:
 				"golang.go",
 				"bradymholt.pgformatter",
 				// optional:
-				// "766b.go-outliner",
+				"42crunch.vscode-openapi",
 				"heaths.vscode-guid",
 				"bungcip.better-toml",
 				"eamodio.gitlens",
-				"casualjim.gotemplate"
-				// "mtxr.sqltools",
+				"casualjim.gotemplate",
+				"yzhang.markdown-all-in-one"
 			]
 		}
 	},
@@ -115,6 +95,7 @@
 	// "shutdownAction": "none",
 	// Uncomment the next line to run commands after the container is created - for example installing git.
 	"postCreateCommand": "go version",
+	// "postCreateCommand": "apt-get update && apt-get install -y git",
 	// Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "development"
+	// "remoteUser": ""
 }

.drone.yml

@@ -30,7 +30,7 @@ alias:
   - &IMAGE_DEPLOY_ID ${DRONE_REPO,,}:${DRONE_COMMIT_SHA}
 
   # Defines which branches will trigger a docker image push our Google Cloud Registry (tags are always published)
-  - &GCR_PUBLISH_BRANCHES [dev, master, aj/pooling-improvements]
+  - &GCR_PUBLISH_BRANCHES [dev, master, aj/pooling-improvements, mr/aj-review]
 
   # Docker registry publish default settings
   - &GCR_REGISTRY_SETTINGS
@@ -133,7 +133,7 @@ pipeline:
     environment:
       IMAGE_TAG: *IMAGE_BUILDER_ID
     commands:
-      - "docker build --target builder-integresql --compress -t $${IMAGE_TAG} ."
+      - "docker build --target builder --compress -t $${IMAGE_TAG} ."
     <<: *WHEN_BUILD_EVENT
 
   "docker build (target integresql)":

.golangci.yml

@@ -0,0 +1,29 @@
+linters:
+  enable:
+    # https://github.com/golangci/golangci-lint#enabled-by-default-linters
+    # Additional linters you want to activate may be specified here...
+
+    # ---
+    # https://github.com/mgechev/revive
+    # replacement for the now deprecated official golint linter, see https://github.com/golang/go/issues/38968
+    - revive
+
+    # ---
+    # https://github.com/maratori/testpackage
+    # used to enforce blackbox testing
+    - testpackage
+
+    # ---
+    # https://github.com/securego/gosec
+    # inspects source code for security problems by scanning the Go AST.
+    - gosec
+
+    # ---
+    # https://github.com/sivchari/tenv
+    # prefer t.Setenv instead of os.Setenv within test code.
+    - tenv
+
+    # ---
+    # https://github.com/polyfloyd/go-errorlint
+    # ensure we are comparing errors via errors.Is, types/values via errors.As and wrap errors with %w.
+    - errorlint

Dockerfile

@@ -18,23 +18,55 @@ ENV MAKEFLAGS "-j 8 --no-print-directory"
 # e.g. stretch=>stretch-pgdg, buster=>buster-pgdg, bullseye=>bullseye-pgdg
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" \
     | tee /etc/apt/sources.list.d/pgdg.list \
-    && apt install curl ca-certificates gnupg \
-    && curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
-
+    && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \
+    | apt-key add -
 
 # Install required system dependencies
 RUN apt-get update \
     && apt-get install -y \
+    #
+    # Mandadory minimal linux packages
+    # Installed at development stage and app stage
+    # Do not forget to add mandadory linux packages to the final app Dockerfile stage below!
+    #
+    # -- START MANDADORY --
+    ca-certificates \
+    # --- END MANDADORY ---
+    #
+    # Development specific packages
+    # Only installed at development stage and NOT available in the final Docker stage
+    # based upon
+    # https://github.com/microsoft/vscode-remote-try-go/blob/master/.devcontainer/Dockerfile
+    # https://raw.githubusercontent.com/microsoft/vscode-dev-containers/master/script-library/common-debian.sh
+    #
+    # icu-devtools: https://stackoverflow.com/questions/58736399/how-to-get-vscode-liveshare-extension-working-when-running-inside-vscode-remote
+    # graphviz: https://github.com/google/pprof#building-pprof
+    # -- START DEVELOPMENT --
+    apt-utils \
+    dialog \
+    openssh-client \
+    less \
+    iproute2 \
+    procps \
+    lsb-release \
     locales \
     sudo \
     bash-completion \
     bsdmainutils \
+    graphviz \
+    xz-utils \
     postgresql-client-12 \
+    icu-devtools \
+    tmux \
+    rsync \
+    # --- END DEVELOPMENT ---
+    #
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# vscode support: LANG must be supported, requires installing the locale package first
-# see https://github.com/Microsoft/vscode/issues/58015
+# env/vscode support: LANG must be supported, requires installing the locale package first
+# https://github.com/Microsoft/vscode/issues/58015
+# https://stackoverflow.com/questions/28405902/how-to-set-the-locale-inside-a-debian-ubuntu-docker-container
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     dpkg-reconfigure --frontend=noninteractive locales && \
     update-locale LANG=en_US.UTF-8
@@ -82,6 +114,25 @@ RUN ARCH="$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)" \
 # https://github.com/uw-labs/lichen/tags
 RUN go install github.com/uw-labs/[email protected]
 
+# watchexec
+# https://github.com/watchexec/watchexec/releases
+RUN mkdir -p /tmp/watchexec \
+    && cd /tmp/watchexec \
+    && wget https://github.com/watchexec/watchexec/releases/download/v1.20.6/watchexec-1.20.6-$(arch)-unknown-linux-musl.tar.xz \
+    && tar xf watchexec-1.20.6-$(arch)-unknown-linux-musl.tar.xz \
+    && cp watchexec-1.20.6-$(arch)-unknown-linux-musl/watchexec /usr/local/bin/watchexec \
+    && rm -rf /tmp/watchexec
+
+# yq
+# https://github.com/mikefarah/yq/releases
+RUN mkdir -p /tmp/yq \
+    && cd /tmp/yq \
+    && ARCH="$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/)" \
+    && wget "https://github.com/mikefarah/yq/releases/download/v4.30.5/yq_linux_${ARCH}.tar.gz" \
+    && tar xzf "yq_linux_${ARCH}.tar.gz" \
+    && cp "yq_linux_${ARCH}" /usr/local/bin/yq \
+    && rm -rf /tmp/yq
+
 # linux permissions / vscode support: Add user to avoid linux file permission issues
 # Detail: Inside the container, any mounted files/folders will have the exact same permissions
 # as outside the container - including the owner user ID (UID) and group ID (GID).
@@ -100,7 +151,6 @@ RUN groupadd --gid $USER_GID $USERNAME \
     && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
     && chmod 0440 /etc/sudoers.d/$USERNAME
 
-
 # vscode support: cached extensions install directory
 # https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
 RUN mkdir -p /home/$USERNAME/.vscode-server/extensions \
@@ -113,7 +163,6 @@ RUN mkdir -p /home/$USERNAME/.vscode-server/extensions \
 # Note that this should be the final step after installing all build deps
 RUN mkdir -p /$GOPATH/pkg && chown -R $USERNAME /$GOPATH
 
-
 # $GOBIN is where our own compiled binaries will live and other go.mod / VSCode binaries will be installed.
 # It should always come AFTER our other $PATH segments and should be earliest targeted in stage "builder",
 # as /app/bin will the shadowed by a volume mount via docker-compose!
@@ -133,15 +182,11 @@ COPY Makefile /app/Makefile
 COPY go.mod /app/go.mod
 COPY go.sum /app/go.sum
 COPY tools.go /app/tools.go
-RUN make modules && make tools
+RUN make modules
+COPY tools.go /app/tools.go
+RUN make tools
 COPY . /app/
-
-### -----------------------
-# --- Stage: builder-integresql
-### -----------------------
-
-FROM builder as builder-integresql
-RUN make build
+RUN make go-build
 
 ### -----------------------
 # --- Stage: integresql
@@ -152,7 +197,7 @@ RUN make build
 # The :debug image provides a busybox shell to enter.
 # https://github.com/GoogleContainerTools/distroless#debug-images
 FROM gcr.io/distroless/base-debian11:debug as integresql
-COPY --from=builder-integresql /app/bin/integresql /
+COPY --from=builder /app/bin/integresql /
 # Note that cmd is not supported with these kind of images, no shell included
 # see https://github.com/GoogleContainerTools/distroless/issues/62
 # and https://github.com/GoogleContainerTools/distroless#entrypoints

Makefile

@@ -4,8 +4,8 @@
 
 # first is default target when running "make" without args
 build: ##- Default 'make' target: go-format, go-build and lint.
-	@$(MAKE) format
-	@$(MAKE) gobuild
+	@$(MAKE) go-format
+	@$(MAKE) go-build
 	@$(MAKE) lint
 
 # useful to ensure that everything gets resetuped from scratch
@@ -22,14 +22,16 @@ info-go: ##- (opt) Prints go.mod updates, module-name and current go version.
 	@go version >> tmp/.info-go
 	@cat tmp/.info-go
 
-format:
-	go fmt
+lint: go-lint  ##- Runs golangci-lint and make check-*.
 
-gobuild:
-	go build -o bin/integresql ./cmd/server
+go-format: ##- (opt) Runs go format.
+	go fmt ./...
 
-lint:
-	golangci-lint run --fast
+go-build: ##- (opt) Runs go build.
+	go build -ldflags $(LDFLAGS) -o bin/integresql ./cmd/server
+
+go-lint: ##- (opt) Runs golangci-lint.
+	golangci-lint run --timeout 5m
 
 bench: ##- Run tests, output by package, print coverage.
 	@go test -benchmem=false -run=./... -bench . github.com/allaboutapps/integresql/tests -race -count=4 -v
@@ -45,7 +47,10 @@ test: ##- Run tests, output by package, print coverage.
 
 # note that we explicitly don't want to use a -coverpkg=./... option, per pkg coverage take precedence
 go-test-by-pkg: ##- (opt) Run tests, output by package.
-	gotestsum --format pkgname-and-test-fails --jsonfile /tmp/test.log -- -race -cover -count=1 -coverprofile=/tmp/coverage.out ./...
+	gotestsum --format pkgname-and-test-fails --format-hide-empty-pkg --jsonfile /tmp/test.log -- -race -cover -count=1 -coverprofile=/tmp/coverage.out ./...
+
+go-test-by-name: ##- (opt) Run tests, output by testname.
+	gotestsum --format testname --jsonfile /tmp/test.log -- -race -cover -count=1 -coverprofile=/tmp/coverage.out ./...
 
 go-test-print-coverage: ##- (opt) Print overall test coverage (must be done after running tests).
 	@printf "coverage "
@@ -176,10 +181,6 @@ LDFLAGS = $(eval LDFLAGS := "\
 # required to ensure make fails if one recipe fails (even on parallel jobs) and on pipefails
 .ONESHELL:
 
-# # normal POSIX bash shell mode
-# SHELL = /bin/bash
-# .SHELLFLAGS = -cEeuo pipefail
-
-# wrapped make time tracing shell, use it via MAKE_TRACE_TIME=true make <target>
-# SHELL = /bin/rksh
-# .SHELLFLAGS = $@
+# normal POSIX bash shell mode
+SHELL = /bin/bash
+.SHELLFLAGS = -cEeuo pipefail

cmd/server/main.go

@@ -2,29 +2,49 @@ package main
 
 import (
 	"context"
-	"log"
+	"errors"
 	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
 	"github.com/allaboutapps/integresql/internal/api"
+	"github.com/allaboutapps/integresql/internal/config"
 	"github.com/allaboutapps/integresql/internal/router"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 )
 
 func main() {
-	s := api.DefaultServerFromEnv()
+
+	cfg := api.DefaultServerConfigFromEnv()
+
+	zerolog.TimeFieldFormat = time.RFC3339Nano
+	zerolog.SetGlobalLevel(cfg.Logger.Level)
+	if cfg.Logger.PrettyPrintConsole {
+		log.Logger = log.Output(zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) {
+			w.TimeFormat = "15:04:05"
+		}))
+	}
+
+	log.Info().Str("version", config.GetFormattedBuildArgs()).Msg("starting...")
+
+	s := api.NewServer(cfg)
 
 	if err := s.InitManager(context.Background()); err != nil {
-		log.Fatalf("Failed to initialize manager: %v", err)
+		log.Fatal().Err(err).Msg("Failed to initialize manager")
 	}
 
 	router.Init(s)
 
 	go func() {
 		if err := s.Start(); err != nil {
-			log.Fatalf("Failed to start server: %v", err)
+			if errors.Is(err, http.ErrServerClosed) {
+				log.Info().Msg("Server closed")
+			} else {
+				log.Fatal().Err(err).Msg("Failed to start server")
+			}
 		}
 	}()
 
@@ -35,7 +55,7 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
-	if err := s.Shutdown(ctx); err != nil && err != http.ErrServerClosed {
-		log.Fatalf("Failed to gracefully shut down server: %v", err)
+	if err := s.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		log.Fatal().Err(err).Msg("Failed to gracefully shut down server")
 	}
 }