diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3b251a6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Added +### Changed +### Fixed +### Docs + +## [1.1.0] - 2022-03-28 + +### Added +- Support for prometheus database + +## [1.0.0] - 2022-02-14 +Initial release! First stable version of GoGeizhalsBot is published as v1.0.0 + +[unreleased]: https://github.com/d-Rickyy-b/GoGeizhalsBot/compare/v1.1.0...HEAD +[1.1.0]: https://github.com/d-Rickyy-b/GoGeizhalsBot/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/d-Rickyy-b/GoGeizhalsBot/tree/v1.0.0 diff --git a/go.mod b/go.mod index 01da8d3..737879d 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4 github.com/PuerkitoBio/goquery v1.8.0 + github.com/VictoriaMetrics/metrics v1.18.1 github.com/davecgh/go-spew v1.1.1 // indirect github.com/glebarez/sqlite v1.3.3 github.com/jinzhu/now v1.1.4 // indirect diff --git a/go.sum b/go.sum index 4ad1db4..323102c 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4 h1:1hKv9DvEwn//VsJgcCGt9+IAq4n+P github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.4/go.mod h1:r815fYWTudnU9JhtsJAxUtuV7QrSgKpChJkfTSMFpfg= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/VictoriaMetrics/metrics v1.18.1 h1:OZ0+kTTto8oPfHnVAnTOoyl0XlRhRkoQrD2n2cOuRw0= +github.com/VictoriaMetrics/metrics v1.18.1/go.mod h1:ArjwVz7WpgpegX/JpB0zpNF2h2232kErkEnzH1sxMmA= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,6 +35,10 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= +github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/wcharczuk/go-chart/v2 v2.1.0 h1:tY2slqVQ6bN+yHSnDYwZebLQFkphK4WNrVwnt7CJZ2I= github.com/wcharczuk/go-chart/v2 v2.1.0/go.mod h1:yx7MvAVNcP/kN9lKXM/NTce4au4DFN99j6i1OwDclNA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/bot/bot.go b/internal/bot/bot.go index d379b19..5d0c9dc 100644 --- a/internal/bot/bot.go +++ b/internal/bot/bot.go @@ -6,10 +6,12 @@ import ( "GoGeizhalsBot/internal/config" "GoGeizhalsBot/internal/database" "GoGeizhalsBot/internal/geizhals" + "GoGeizhalsBot/internal/prometheus" "fmt" "log" "net/http" "net/url" + "time" "github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/message" @@ -218,6 +220,7 @@ func showPriceagentDetail(b *gotgbot.Bot, ctx *ext.Context) error { }, } + // Check if the initial message contained a photo, if yes, we're coming from the price history graph if len(cb.Message.Photo) > 0 { bot.DeleteMessage(ctx.EffectiveChat.Id, cb.Message.MessageId) @@ -344,6 +347,7 @@ func deletePriceagentHandler(b *gotgbot.Bot, ctx *ext.Context) error { } func newUserHandler(_ *gotgbot.Bot, ctx *ext.Context) error { + prometheus.TotalUserInteractions.Inc() // Create user in databse if they don't exist already if !ctx.EffectiveSender.IsUser() { return nil @@ -480,5 +484,22 @@ func Start(botConfig config.Config) { log.Printf("Bot has been started as @%s...\n", bot.User.Username) + if botConfig.Prometheus.Enabled { + // Periodically update the metrics from the database + go func() { + for { + prometheus.TotalUniquePriceagentsValue = database.GetPriceAgentCount() + prometheus.TotalUniqueUsersValue = database.GetUserCount() + prometheus.TotalUniqueWishlistPriceagentsValue = database.GetPriceAgentWishlistCount() + prometheus.TotalUniqueProductPriceagentsValue = database.GetPriceAgentProductCount() + time.Sleep(time.Second * 60) + } + }() + + exportAddr := fmt.Sprintf("%s:%d", botConfig.Prometheus.ExportIP, botConfig.Prometheus.ExportPort) + log.Printf("Starting prometheus exporter on %s...\n", exportAddr) + prometheus.StartPrometheusExporter(exportAddr) + } + updater.Idle() } diff --git a/internal/bot/notification.go b/internal/bot/notification.go index 6e88670..187e83f 100644 --- a/internal/bot/notification.go +++ b/internal/bot/notification.go @@ -4,6 +4,7 @@ import ( "GoGeizhalsBot/internal/bot/models" "GoGeizhalsBot/internal/database" "GoGeizhalsBot/internal/geizhals" + "GoGeizhalsBot/internal/prometheus" "fmt" "log" "time" @@ -23,7 +24,7 @@ func updateEntityPrices() { // For each price agent, update prices and store updated prices in the entity in the database. // Also update price history with the new prices. for _, entity := range allEntities { - log.Println("Updating prices for:", entity.Name) + log.Println("Updating prices for:", entity.URL) // If there are two price agents with the same entity, we currently fetch it twice updatedEntity, updateErr := geizhals.UpdateEntity(entity) @@ -84,6 +85,7 @@ func notifyUsers(priceAgent models.PriceAgent, oldEntity, updatedEntity geizhals } log.Println("Sending notification to user:", user.ID) + prometheus.PriceagentNotifications.Inc() markup := gotgbot.InlineKeyboardMarkup{ InlineKeyboard: [][]gotgbot.InlineKeyboardButton{ diff --git a/internal/bot/pricehistory.go b/internal/bot/pricehistory.go index 51000fe..ceb2eb0 100644 --- a/internal/bot/pricehistory.go +++ b/internal/bot/pricehistory.go @@ -4,6 +4,7 @@ import ( "GoGeizhalsBot/internal/bot/models" "GoGeizhalsBot/internal/database" "GoGeizhalsBot/internal/geizhals" + "GoGeizhalsBot/internal/prometheus" "bytes" "fmt" "io" @@ -145,6 +146,7 @@ func getPriceagentFromContext(ctx *ext.Context) (models.PriceAgent, error) { // renderChart renders a price history chart to the given writer. func renderChart(priceagent models.PriceAgent, history geizhals.PriceHistory, since time.Time, w io.Writer) { + prometheus.GraphsRendered.Inc() darkFontColor := drawing.ColorFromHex("c2c2c2") fontColor := darkFontColor diff --git a/internal/bot/version.go b/internal/bot/version.go index 7abe4ed..5ce40a8 100644 --- a/internal/bot/version.go +++ b/internal/bot/version.go @@ -1,3 +1,3 @@ package bot -var version = "1.0.0-dev (compiled manually)" +var version = "1.1.0-dev (compiled manually)" diff --git a/internal/database/database.go b/internal/database/database.go index 198cf39..f17df32 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -53,11 +53,41 @@ func GetPriceAgentCountForUser(userID int64) int64 { return count } +func GetPriceAgentCount() int64 { + var count int64 + db.Model(&models.PriceAgent{}).Count(&count) + return count +} + +func GetPriceAgentProductCount() int64 { + var count int64 + db.Model(&models.PriceAgent{}).Joins("JOIN entities on price_agents.entity_id = entities.id").Where("entities.type = ?", geizhals.Product).Count(&count) + return count +} + +func GetPriceAgentWishlistCount() int64 { + var count int64 + db.Model(&models.PriceAgent{}).Joins("JOIN entities on price_agents.entity_id = entities.id").Where("entities.type = ?", geizhals.Wishlist).Count(&count) + return count +} + +func GetUserCount() int64 { + var count int64 + db.Model(&models.User{}).Count(&count) + return count +} + func CreateUser(user models.User) error { tx := db.Create(&user) return tx.Error } +func GetAllUsers() []models.User { + var users []models.User + db.Find(&users) + return users +} + func DeletePriceAgentForUser(priceAgent models.PriceAgent) error { log.Println("Delete priceagent!") diff --git a/internal/geizhals/geizhals.go b/internal/geizhals/geizhals.go index d40e991..d3d9a80 100644 --- a/internal/geizhals/geizhals.go +++ b/internal/geizhals/geizhals.go @@ -1,6 +1,7 @@ package geizhals import ( + "GoGeizhalsBot/internal/prometheus" "fmt" "log" "net/http" @@ -72,9 +73,11 @@ func downloadHTML(entityURL string) (*goquery.Document, int, error) { log.Println("Using proxy: ", proxyURL) } + prometheus.GeizhalsHTTPRequests.Inc() resp, getErr := httpClient.Get(entityURL) if getErr != nil { log.Println(getErr) + prometheus.HttpErrors.Inc() return nil, 0, fmt.Errorf("error while downloading content from Geizhals: %w", getErr) } // Cleanup when this function ends diff --git a/internal/prometheus/prometheus.go b/internal/prometheus/prometheus.go new file mode 100644 index 0000000..82d4692 --- /dev/null +++ b/internal/prometheus/prometheus.go @@ -0,0 +1,45 @@ +package prometheus + +import ( + "net/http" + + "github.com/VictoriaMetrics/metrics" +) + +var ( + TotalUniqueUsersValue int64 + totalUniqueUsers = metrics.NewGauge("gogeizhalsbot_unique_users_total", func() float64 { + return float64(TotalUniqueUsersValue) + }) + + TotalUniquePriceagentsValue int64 + totalUniquePriceagents = metrics.NewGauge("gogeizhalsbot_unique_priceagents", func() float64 { + return float64(TotalUniquePriceagentsValue) + }) + + TotalUniqueProductPriceagentsValue int64 + totalUniqueProductPriceagents = metrics.NewGauge("gogeizhalsbot_unique_priceagents{type=\"product\"}", func() float64 { + return float64(TotalUniqueProductPriceagentsValue) + }) + + TotalUniqueWishlistPriceagentsValue int64 + totalUniqueWishlistPriceagents = metrics.NewGauge("gogeizhalsbot_unique_priceagents{type=\"wishlist\"}", func() float64 { + return float64(TotalUniqueWishlistPriceagentsValue) + }) + + TotalUserInteractions = metrics.NewCounter("gogeizhalsbot_user_interactions_total") + GeizhalsHTTPRequests = metrics.NewCounter("gogeizhalsbot_geizhals_http_requests_total") + PriceagentNotifications = metrics.NewCounter("gogeizhalsbot_priceagent_notifications_total") + HttpErrors = metrics.NewCounter("gogeizhalsbot_http_errors_total") + GraphsRendered = metrics.NewCounter("gogeizhalsbot_graphs_rendered_total") +) + +// var backgroundUpdateChecks = metrics.NewSummary("gogeizhalsbot_total_requests") + +func StartPrometheusExporter(addr string) { + // Expose the registered metrics at `/metrics` path. + http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) { + metrics.WritePrometheus(w, true) + }) + http.ListenAndServe(addr, nil) +}