add go source files

.gitignore

@@ -0,0 +1 @@
+.idea

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023 Marcel Hencke
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -0,0 +1,51 @@
+# nglog - Formats (*php-fpm* + *nginx*) logs
+
+**nglog**  is a cli program, which splits and formats typical PHP error messages from a nginx error log file.
+
+
+### Without:
+```
+user@server:~# tail /var/log/nginx/example.com_error.log
+
+2023/07/03 03:03:37 [error] 396782#396782: *300397 FastCGI sent in stderr: "PHP message: PHP Warning:  Undefined array key "key0" in /var/www/example.com/index.php on line 12PHP message: PHP Warning:  Undefined array key "key1" in /var/www/example.com/index.php on line 13" while reading upstream, client: 127.0.0.1, server: example.com, request: "GET / HTTP/2.0", upstream: "fastcgi://unix:/run/php/php8.0-fpm.sock:", host: "example.com"
+```
+
+### With nglog:
+```
+user@server:~# tail /var/log/nginx/example.com_error.log | nglog
+
+2023/07/03 03:03:37 [error] 396782#396782: *300397 FastCGI sent in stderr: "PHP Warning:  Undefined array key "key0" in /var/www/example.com/index.php on line 12" while reading upstream, client: 127.0.0.1, server: example.com, request: "GET / HTTP/2.0", upstream: "fastcgi://unix:/run/php/php8.0-fpm.sock:", host: "example.com"
+2023/07/03 03:03:37 [error] 396782#396782: *300397 FastCGI sent in stderr: "PHP Warning:  Undefined array key "key1" in /var/www/example.com/index.php on line 13" while reading upstream, client: 127.0.0.1, server: example.com, request: "GET / HTTP/2.0", upstream: "fastcgi://unix:/run/php/php8.0-fpm.sock:", host: "example.com"
+```
+
+```
+user@server:~# tail /var/log/nginx/example.com_error.log | nglog -t "%ts% - %php%"
+
+2023/07/03 03:03:37 - PHP Warning:  Undefined array key "key0" in /var/www/example.com/index.php on line 12
+2023/07/03 03:03:37 - PHP Warning:  Undefined array key "key1" in /var/www/example.com/index.php on line 13
+```
+
+
+
+# Usage
+```
+log file as argument:
+user@server:~# nglog /var/log/nginx/example.com_error.log
+
+tail reads log file and transfer data via a pipe to nglog:
+user@server:~# tail -f /var/log/nginx/example.com_error.log | nglog
+
+custom template for log file:
+user@server:~# nglog -t "%ts% - %php% - %ng_upstream%" /var/log/nginx/example.com_error.log
+```
+
+
+[//]: # (TODO: all params)
+
+
+# Credits
+https://github.com/napicella/go-linux-pipes
+https://github.com/spf13/cobra
+
+# License
+nglog is released under the Apache 2.0 license. See [LICENSE](LICENSE)

cmd.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"nglog/config"
+	debug "nglog/debugPrint"
+	"nglog/nginxErrorLogFormatter"
+	"os"
+)
+
+func LoadData() error {
+	if isInputFromPipe() {
+		debug.Println("Reading data from pipe")
+		return processData(os.Stdin, os.Stdout)
+	} else {
+		debug.Println("Loading data from file: " + config.Flags.FilePath)
+		file, e := getFile()
+		if e != nil {
+			return e
+		}
+		defer file.Close()
+		return processData(file, os.Stdout)
+	}
+}
+
+func processData(r io.Reader, w io.Writer) error {
+	formatter := nginxErrorLogFormatter.New(w)
+
+	scanner := bufio.NewScanner(bufio.NewReader(r))
+	rowCounter := 0
+	for scanner.Scan() {
+		rowCounter++
+		e := formatter.ReadBufferLine(scanner.Text())
+		if e != nil {
+			return e
+		}
+		if config.Flags.DebugMode && config.Flags.DebugMaxLines > 0 && rowCounter >= config.Flags.DebugMaxLines {
+			return nil
+		}
+	}
+	return nil
+}
+
+func isInputFromPipe() bool {
+	fi, _ := os.Stdin.Stat()
+	return fi.Mode()&os.ModeCharDevice == 0
+}
+
+func getFile() (*os.File, error) {
+	if config.Flags.FilePath == "" {
+		return nil, errors.New("please input a file path")
+	}
+	if !fileExists(config.Flags.FilePath) {
+		return nil, errors.New("the file provided does not exist")
+	}
+	file, e := os.Open(config.Flags.FilePath)
+	if e != nil {
+		return nil, errors.New("unable to read the file " + config.Flags.FilePath + " : " + e.Error())
+	}
+	return file, nil
+}
+
+func fileExists(filepath string) bool {
+	info, e := os.Stat(filepath)
+	if os.IsNotExist(e) {
+		return false
+	}
+	return !info.IsDir()
+}

config/flags.go

@@ -0,0 +1,12 @@
+package config
+
+var Flags struct {
+	FilePath             string
+	Template             string
+	DebugMode            bool
+	DebugMaxLines        int
+	LogLineCompleteRegex string
+	LogLineFastCGIRegex  string
+	LogLinePhpMsgSplit   string
+	NginxVarRegex        string
+}

debugPrint/print.go

@@ -0,0 +1,14 @@
+package debugPrint
+
+import (
+	"fmt"
+)
+
+var Println func(a ...any)
+
+func PrintNoop(a ...any) {}
+
+func PrintFmt(a ...any) {
+	fmt.Print("DEBUG: ")
+	fmt.Println(a...)
+}

go.mod

@@ -0,0 +1,10 @@
+module nglog
+
+go 1.20
+
+require github.com/spf13/cobra v1.7.0
+
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+)

go.sum

@@ -0,0 +1,10 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+	"github.com/spf13/cobra"
+	"nglog/config"
+	debug "nglog/debugPrint"
+	"os"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "nglog [flags] [LOG_FILE]",
+	Short: "Formats (php-fpm + nginx) logs",
+	Long:  `nglog read text lines from LOG_FILE or from PIPE. This lines gets buffered until the LogLineCompleteRegex matches. After this nglog trys to parse a typical php error line in a nginx error log file (LogLineFastCGIRegex). If this parsing is successful, nglog uses this data to print a new log line defined by the template. `,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) > 0 {
+			config.Flags.FilePath = args[0]
+		}
+		debug.Println = debug.PrintNoop
+		if config.Flags.DebugMode {
+			debug.Println = debug.PrintFmt
+		}
+		return LoadData()
+	},
+}
+
+func main() {
+	rootCmd.Flags().StringVarP(
+		&config.Flags.Template,
+		"template",
+		"t",
+		"%raw%",
+		"This is a template format for the new line. The key must be enclosed in %\n"+
+			"Possible keys:\n"+
+			"- raw: no further line manipulation\n"+
+			"- prefix: first match group from LogLineFastCGIRegex\n"+
+			"- suffix: third match group from LogLineFastCGIRegex\n"+
+			"- ts: timestamp\n"+
+			"- php: PHP message\n"+
+			"- ng_xxx: nginx var, e.g. ng_server, ng_upstream ...")
+
+	rootCmd.Flags().StringVar(
+		&config.Flags.LogLineCompleteRegex,
+		"overwriteLogLineCompleteRegex",
+		"\" while reading[A-z ]* upstream",
+		"This RegEx tests whether the lines read are a complete log line.\n")
+	rootCmd.Flags().StringVar(
+		&config.Flags.LogLineFastCGIRegex,
+		"overwriteLogLineFastCGIRegex",
+		"^(\\d{4}[\\/-]\\d{2}[\\/-]\\d{2} \\d{2}:\\d{2}:\\d{2} \\[\\w+\\] .+ FastCGI sent in stderr: \")([\\s\\S]*)(\" while reading[A-z ]* upstream(?:, \\w+: \"?.+\"?)*)$",
+		"This RegEx tests whether the log line is a FastCGI log line.\nThis expression are divided in 3 matching groups: prefix, body, suffix.\nPrefix: Timestamp and FastCGI identifier.\nBody: Contains the PHP messages.\nSuffix: Some nginx properties, like client, server, upstream, host, etc.\n")
+	rootCmd.Flags().StringVar(
+		&config.Flags.LogLinePhpMsgSplit,
+		"overwriteLogLinePhpMsgSplit",
+		"PHP message: ",
+		"This string is the split value for PHP messages.\n")
+	rootCmd.Flags().StringVar(
+		&config.Flags.NginxVarRegex,
+		"overwriteNginxVarRegex",
+		", (\\w+): (\"?[^,]+\"?)",
+		"This RegEx finds the nginx vars in the line suffix.\n")
+
+	rootCmd.PersistentFlags().BoolVarP(
+		&config.Flags.DebugMode,
+		"debugMode",
+		"d",
+		false, "Print out debug logs. Helpful when defining custom regex.")
+	rootCmd.Flags().IntVar(
+		&config.Flags.DebugMaxLines,
+		"debugMaxLines",
+		0,
+		"Only read x lines of input data. Only effective when debugMode is enabled and x > 0. ")
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println("Error message: " + err.Error() + ".")
+		os.Exit(1)
+	}
+}

nginxErrorLogFormatter/formater.go

@@ -0,0 +1,109 @@
+package nginxErrorLogFormatter
+
+import (
+	"fmt"
+	"io"
+	"nglog/config"
+	debug "nglog/debugPrint"
+	"regexp"
+	"strings"
+	"sync"
+)
+
+type NginxErrorLogFormatter struct {
+	bufferMutex           sync.RWMutex
+	buffer                string
+	ioWriter              io.Writer
+	rLogLineCompleteRegex *regexp.Regexp
+	rLogLineFastCGIRegex  *regexp.Regexp
+	rVarRegexNg           *regexp.Regexp
+	rVarRegexTs           *regexp.Regexp
+}
+
+func New(w io.Writer) *NginxErrorLogFormatter {
+	ptr := &NginxErrorLogFormatter{
+		buffer:   "",
+		ioWriter: w,
+	}
+	ptr.rLogLineCompleteRegex, _ = regexp.Compile(config.Flags.LogLineCompleteRegex)
+	ptr.rLogLineFastCGIRegex, _ = regexp.Compile(config.Flags.LogLineFastCGIRegex)
+	ptr.rVarRegexNg, _ = regexp.Compile(config.Flags.NginxVarRegex)
+	ptr.rVarRegexTs, _ = regexp.Compile("\\d{4}[\\/-]\\d{2}[\\/-]\\d{2} \\d{2}:\\d{2}:\\d{2}")
+	return ptr
+}
+
+func (this *NginxErrorLogFormatter) ReadBufferLine(line string) error {
+	debug.Println("ReadBufferLine: " + line)
+	this.buffer += line
+	if this.rLogLineCompleteRegex.MatchString(this.buffer) {
+		debug.Println("ReadBufferLine: LINE is complete (matches LogLineCompleteRegex)")
+		e := this.parseFastCGILogLine(this.buffer)
+		this.buffer = ""
+		return e
+	}
+	this.buffer += "\n"
+	return nil
+}
+
+func (this *NginxErrorLogFormatter) parseFastCGILogLine(logLine string) error {
+	if this.rLogLineFastCGIRegex.MatchString(logLine) {
+		matches := this.rLogLineFastCGIRegex.FindStringSubmatch(logLine)
+		debug.Println("FastCGI Parser match LogLineFastCGIRegex")
+		debug.Println("- PREFIX: " + matches[1])
+		debug.Println("- BODY: " + matches[2])
+		debug.Println("- SUFFIX: " + matches[3])
+		return this.splitPhpLine(matches[1], matches[2], matches[3])
+	}
+	debug.Println("FastCGI Parser could NOT match LogLineFastCGIRegex: " + logLine)
+	return this.writeOut(logLine)
+}
+
+func (this *NginxErrorLogFormatter) splitPhpLine(prefix, phpMsgs, suffix string) error {
+	phpMsgsArr := strings.Split(phpMsgs, config.Flags.LogLinePhpMsgSplit)
+	for _, phpMsg := range phpMsgsArr {
+		if phpMsg == "" {
+			continue
+		}
+		debug.Println("PHP message after split: " + phpMsg)
+		err := this.writeOut(this.reformatPhpLine(prefix, phpMsg, suffix))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (this *NginxErrorLogFormatter) reformatPhpLine(prefix, phpMsg, suffix string) string {
+	if config.Flags.Template == "%raw%" {
+		return prefix + phpMsg + suffix
+	}
+
+	dataTemplate := config.Flags.Template
+	debug.Println("PHP format: " + dataTemplate)
+	data := map[string]string{}
+	data["raw"] = prefix + phpMsg + suffix
+	data["prefix"] = prefix
+	data["suffix"] = suffix
+	data["php"] = phpMsg
+
+	data["ts"] = this.rVarRegexTs.FindString(prefix)
+	debug.Println("PHP format ts match: " + data["ts"])
+
+	ngVars := this.rVarRegexNg.FindAllStringSubmatch(suffix, -1)
+	for _, ngVar := range ngVars {
+		if len(ngVar) > 2 {
+			debug.Println("PHP format ng_" + ngVar[1] + " match: " + ngVar[2])
+			data["ng_"+ngVar[1]] = ngVar[2]
+		}
+	}
+
+	for k, v := range data {
+		dataTemplate = strings.ReplaceAll(dataTemplate, "%"+k+"%", v)
+	}
+	return dataTemplate
+}
+
+func (this *NginxErrorLogFormatter) writeOut(data string) error {
+	_, e := fmt.Fprintln(this.ioWriter, data)
+	return e
+}