Skip to content

Commit

Permalink
Add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
mtodor committed Nov 13, 2024
1 parent 2d6a385 commit 6619bc3
Show file tree
Hide file tree
Showing 3 changed files with 242 additions and 0 deletions.
53 changes: 53 additions & 0 deletions cmd/flakechecker/flake_config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package main

import (
"github.com/stretchr/testify/assert"
"testing"
)

func TestLoadFlakeConfigFile(t *testing.T) {
samples := []struct {
name string
fileName string

expectError bool
expectConfig []*flakeCheckerRecord
}{
{
name: "no config file",
fileName: "no_config.json",
expectError: true,
expectConfig: nil,
},
{
name: "valid config file",
fileName: "testdata/flake-config.json",
expectError: false,
expectConfig: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{
MatchJobName: "pr-.*",
RatioJobName: "main-branch-tests",
TestNameRegex: "TestLoadFlakeConf.*",
Classname: "TestLoadFlakeConfigFile",
RatioThreshold: 5,
}),
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{
MatchJobName: "pull-request-tests",
RatioJobName: "main-branch-tests",
TestNameRegex: "TestLoadFlakeConfigFile",
Classname: "TestLoadFlakeConfigFile",
RatioThreshold: 10,
}),
},
},
}

for _, sample := range samples {
t.Run(sample.name, func(tt *testing.T) {
config, err := loadFlakeConfigFile(sample.fileName)

assert.Equal(tt, sample.expectError, err != nil)
assert.Equal(tt, sample.expectConfig, config)
})
}
}
171 changes: 171 additions & 0 deletions cmd/flakechecker/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
package main

import (
"github.com/pkg/errors"
"github.com/stackrox/junit2jira/pkg/testcase"
"github.com/stretchr/testify/assert"
"testing"
)

type biqQueryClientMock struct {
getRatioForTest func(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error)
}

func (c *biqQueryClientMock) GetRatioForTest(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error) {
if c.getRatioForTest != nil {
return c.getRatioForTest(flakeTestConfig, testName)
}

// By default, fail. In most cases, we will not reach this part in tests.
return 0, 0, errors.New("fail")
}

func getRatioForTestNoFailures(_ *flakeCheckerRecord, _ string) (int, int, error) {
return totalRunsLimit, 0, nil
}

func getRatioForTestAllFailures(_ *flakeCheckerRecord, _ string) (int, int, error) {
return totalRunsLimit, 50, nil
}

func TestCheckFailedTests(t *testing.T) {
p := flakeCheckerParams{jobName: "test-job"}

samples := map[string]struct {
bqClient biqQueryClient
failedTests []testcase.TestCase
flakeConfigs []*flakeCheckerRecord
expectError bool
expectErrorStr string
}{
"no failed tests": {
bqClient: &biqQueryClientMock{},
failedTests: []testcase.TestCase{},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job1-", TestNameRegex: "test name"}),
},
expectError: false,
},
"no config match - job name": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestNoFailures,
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job-1", TestNameRegex: "test", Classname: "class"}),
},
expectError: true,
expectErrorStr: errDescNoMatch,
},
"no config match - test name": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestNoFailures,
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "wrong-test", Classname: "class"}),
},
expectError: true,
expectErrorStr: errDescNoMatch,
},
"no config match - classname": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestNoFailures,
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "wrong-class"}),
},
expectError: true,
expectErrorStr: errDescNoMatch,
},
"unable to fetch ratio": {
bqClient: &biqQueryClientMock{
getRatioForTest: func(_ *flakeCheckerRecord, _ string) (int, int, error) {
return 0, 0, errors.New("fail")
},
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
},
expectError: true,
expectErrorStr: errDescGetRatio,
},
"total runs below limit": {
bqClient: &biqQueryClientMock{
getRatioForTest: func(_ *flakeCheckerRecord, _ string) (int, int, error) {
return totalRunsLimit - 1, 0, nil
},
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
},
expectError: true,
expectErrorStr: errDescShortHistory,
},
"fail ratio below threshold": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestNoFailures,
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
},
expectError: false,
},
"fail ratio above threshold": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestAllFailures,
},
failedTests: []testcase.TestCase{{Name: "test", Classname: "class"}},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
},
expectError: true,
expectErrorStr: errDescAboveThreshold,
},
"fail ratio below threshold - multiple tests": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestNoFailures,
},
failedTests: []testcase.TestCase{
{Name: "test", Classname: "class"},
{Name: "test-1", Classname: "class-1"},
},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test", Classname: "class", RatioThreshold: 1}),
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test-1", Classname: "class-1", RatioThreshold: 1}),
},
expectError: false,
},
"fail ratio above threshold - multiple tests": {
bqClient: &biqQueryClientMock{
getRatioForTest: getRatioForTestAllFailures,
},
failedTests: []testcase.TestCase{
{Name: "test-ratio-below", Classname: "class"},
{Name: "test-ratio-above", Classname: "class"},
},
flakeConfigs: []*flakeCheckerRecord{
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-below", Classname: "class", RatioThreshold: 90}),
newFlakeCheckerRecordMust(&flakeCheckerRecordConfig{MatchJobName: "test-job", TestNameRegex: "test-ratio-above", Classname: "class", RatioThreshold: 10}),
},
expectError: true,
expectErrorStr: errDescAboveThreshold,
},
}

for sampleName, sample := range samples {
t.Run(sampleName, func(tt *testing.T) {
err := p.checkFailedTests(sample.bqClient, sample.failedTests, sample.flakeConfigs)

if sample.expectError {
assert.ErrorContains(tt, err, sample.expectErrorStr)
} else {
assert.NoError(tt, err)
}
})
}
}
18 changes: 18 additions & 0 deletions cmd/flakechecker/testdata/flake-config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[
{
"_comment": "Config with regex",
"match_job_name": "pr-.*",
"ratio_job_name": "main-branch-tests",
"test_name_regex": "TestLoadFlakeConf.*",
"classname": "TestLoadFlakeConfigFile",
"ratio_threshold": 5
},
{
"_comment": "Config with flat values",
"match_job_name": "pull-request-tests",
"ratio_job_name": "main-branch-tests",
"test_name_regex": "TestLoadFlakeConfigFile",
"classname": "TestLoadFlakeConfigFile",
"ratio_threshold": 10
}
]

0 comments on commit 6619bc3

Please sign in to comment.