tests: Initial Integration Tests

Heavily inspired by the already existing integration tests from Icinga
DB, the icinga-testing project was extended to support Icinga
Notifications[0].

Based on those changes, an initial integration test was implemented for
Icinga Notifications, launching and configuring PostgreSQL, Icinga 2,
and Icinga Notifications. Afterwards, a webhook channel is used to catch
notifications.

[0] Icinga/icinga-testing#27

.github/workflows/integration-tests.yml

@@ -0,0 +1,34 @@
+name: Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+      - 'support/*'
+      - 'init-integration-tests' # TODO: remove PR testing branch
+      - 'x-integration-tests'    # TODO: remove PR testing branch
+  pull_request: {}
+  schedule:
+    - cron: '42 23 * * *'
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: stable
+      - name: Run Integration Tests
+        run: ./test.sh
+        working-directory: tests/
+      - name: Compress Debug Log
+        if: ${{ always() }}
+        run: xz -9 ./tests/out/debug.log
+      - name: Upload Debug Log
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: debug.log.xz
+          path: ./tests/out/debug.log.xz
+          retention-days: 1

Dockerfile

@@ -20,6 +20,9 @@ FROM docker.io/library/alpine
 COPY --from=build /src/icinga-notifications/bin/icinga-notifications-daemon /usr/bin/icinga-notifications-daemon
 COPY --from=build /src/icinga-notifications/bin/channel /usr/libexec/icinga-notifications/channel
 
+RUN mkdir /etc/icinga-notifications/
+COPY config.example.yml /etc/icinga-notifications/config.yml
+
 RUN apk add tzdata
 
 ARG username=notifications

tests/go.mod

@@ -0,0 +1,47 @@
+module github.com/icinga/icinga-notifications/tests
+
+go 1.21
+
+replace (
+	github.com/icinga/icinga-notifications => ../
+	github.com/icinga/icinga-testing => github.com/icinga/icinga-testing v0.0.0-20240118120311-bf2ccd424307
+)
+
+require (
+	github.com/icinga/icinga-notifications v0.0.0-20240102102116-0d6f7271c116
+	github.com/icinga/icinga-testing v0.0.0-20240112095229-18da8922599a
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/Icinga/go-libs v0.0.0-20220420130327-ef58ad52edd8 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/go-redis/redis/v8 v8.11.5 // indirect
+	github.com/go-sql-driver/mysql v1.7.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/icinga/icingadb v1.1.1-0.20230418113126-7c4b947aad3a // indirect
+	github.com/lib/pq v1.10.9 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/tools v0.16.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

tests/main_test.go

@@ -0,0 +1,39 @@
+package notifications_test
+
+import (
+	"github.com/icinga/icinga-testing"
+	"github.com/icinga/icinga-testing/services"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+var it *icingatesting.IT
+
+func TestMain(m *testing.M) {
+	it = icingatesting.NewIT()
+	defer it.Cleanup()
+
+	m.Run()
+}
+
+func getDatabase(t testing.TB) services.RelationalDatabase {
+	rdb := getEmptyDatabase(t)
+	rdb.ImportIcingaNotificationsSchema()
+
+	db, err := sqlx.Open(rdb.Driver(), rdb.DSN())
+	require.NoError(t, err, "SQL database open")
+	defer func() { _ = db.Close() }()
+
+	_, err = db.Exec(`
+		INSERT INTO source (id, type, name, listener_password_hash)
+		VALUES (1, 'icinga2', 'Icinga 2', '$2y$10$QU8bJ7cpW1SmoVQ/RndX5O2J5L1PJF7NZ2dlIW7Rv3zUEcbUFg3z2')`)
+	require.NoError(t, err, "populating source table failed")
+
+	return rdb
+}
+
+func getEmptyDatabase(t testing.TB) services.RelationalDatabase {
+	// Currently, PostgreSQL is the only supported database backend.
+	return it.PostgresqlDatabaseT(t)
+}

tests/notification_roundtrip_test.go

@@ -0,0 +1,80 @@
+package notifications_test
+
+import (
+	"encoding/json"
+	"github.com/icinga/icinga-notifications/pkg/plugin"
+	"github.com/icinga/icinga-testing/utils/eventually"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+	"net/http"
+	"testing"
+	"time"
+)
+
+// TestNotificationRoundTrip instructs an Icinga 2 node to send a notification back for further inspection.
+func TestNotificationRoundTrip(t *testing.T) {
+	rdb := getDatabase(t)
+	notifications := it.IcingaNotificationsInstanceT(t, rdb)
+	icinga := it.Icinga2NodeT(t, "master")
+	icinga.EnableIcingaNotifications(notifications)
+	require.NoError(t, icinga.Reload(), "icinga.Reload()")
+
+	db, err := sqlx.Open(rdb.Driver(), rdb.DSN())
+	require.NoError(t, err, "SQL database open")
+	defer func() { require.NoError(t, db.Close(), "db.Cleanup") }()
+
+	webhookRec := it.IcingaNotificationsWebhookReceiverInstanceT(t)
+	webhookRecReqCh := make(chan plugin.NotificationRequest)
+	webhookRec.Handler = func(writer http.ResponseWriter, request *http.Request) {
+		var notReq plugin.NotificationRequest
+		require.NoError(t, json.NewDecoder(request.Body).Decode(&notReq), "decoding NotificationRequest from request body")
+		require.NoError(t, request.Body.Close(), "closing request body")
+		webhookRecReqCh <- notReq
+		http.Error(writer, "forwarded", http.StatusOK)
+	}
+
+	t.Run("configure channel in database", func(t *testing.T) {
+		eventually.Require(t, func(t require.TestingT) {
+			var channelCount int
+			err := db.QueryRow(`SELECT COUNT(*) FROM available_channel_type WHERE type = 'webhook'`).Scan(&channelCount)
+			require.NoError(t, err, "SQL SELECT FROM available_channel_type query")
+			require.Equal(t, 1, channelCount, "webhook type missing from available_channel_type")
+		}, 10*time.Second, time.Second)
+
+		_, err := db.Exec(`
+			INSERT INTO channel (id, name, type, config)
+			VALUES (1, 'webhook', 'webhook', '{"method":"POST","url_template":"http:\/\/` + webhookRec.ListenAddr + `\/","request_body_template":"{{json .}}"}');
+
+			INSERT INTO contact (id, full_name, username, default_channel_id, color)
+			VALUES (1, 'icingaadmin', 'icingaadmin', 1, '#000000');
+
+			INSERT INTO rule (id, name, is_active) VALUES (1, 'webhook', 'y');
+			INSERT INTO rule_escalation (id, rule_id, position) VALUES (1, 1, 1);
+			INSERT INTO rule_escalation_recipient (id, rule_escalation_id, contact_id, channel_id) VALUES (1, 1, 1, 1);`)
+		require.NoError(t, err, "populating tables failed")
+	})
+
+	t.Run("create icinga objects", func(t *testing.T) {
+		client := icinga.ApiClient()
+		client.CreateObject(t, "checkcommands", "failure-check", map[string]any{
+			"templates": []any{"plugin-check-command"},
+			"attrs":     map[string]any{"command": []string{"/bin/false"}},
+		})
+		client.CreateHost(t, "test-host", map[string]any{
+			"attrs": map[string]any{"check_command": "failure-check"},
+		})
+		client.CreateService(t, "test-host", "test-service", map[string]any{
+			"attrs": map[string]any{"check_command": "failure-check"},
+		})
+	})
+
+	t.Run("read notification back from channel", func(t *testing.T) {
+		select {
+		case notReq := <-webhookRecReqCh:
+			require.Contains(t, notReq.Object.Name, "test-", "object name must contain test prefix")
+
+		case <-time.After(5 * time.Minute):
+			require.Fail(t, "no notification was received")
+		}
+	})
+}

tests/test.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -eux
+
+ICINGA_TESTING_NOTIFICATIONS_IMAGE="icinga-notifications:latest"
+ICINGA_TESTING_ICINGA_NOTIFICATIONS_SHARED_DIR="$(realpath out)"
+ICINGA_TESTING_ICINGA_NOTIFICATIONS_SCHEMA_PGSQL="$(realpath ../schema/pgsql/schema.sql)"
+export ICINGA_TESTING_NOTIFICATIONS_IMAGE
+export ICINGA_TESTING_ICINGA_NOTIFICATIONS_SHARED_DIR
+export ICINGA_TESTING_ICINGA_NOTIFICATIONS_SCHEMA_PGSQL
+
+docker build -t "$ICINGA_TESTING_NOTIFICATIONS_IMAGE" ..
+
+test -d ./out && rm -r ./out
+mkdir ./out
+
+go test -o ./out/icinga-notifications-test -c .
+
+exec ./out/icinga-notifications-test -icingatesting.debuglog ./out/debug.log -test.v