Skip to content

Commit

Permalink
change mtbf definition, support mean time in hours and minutes
Browse files Browse the repository at this point in the history
  • Loading branch information
misgod-yy committed Jan 2, 2020
1 parent 90c9832 commit 175ec88
Show file tree
Hide file tree
Showing 17 changed files with 116 additions and 109 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ kube-monkey works on an opt-in model and will only schedule terminations for Kub
Opt-in is done by setting the following labels on a k8s app:

**`kube-monkey/enabled`**: Set to **`"enabled"`** to opt-in to kube-monkey
**`kube-monkey/mtbf`**: Mean time between failure (in days). For example, if set to **`"3"`**, the k8s app can expect to have a Pod
killed approximately every third weekday.
**`kube-monkey/mtbf`**: Mean time between failure (in hours/minutes etc. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".). For example, if set to **`"3h"`**, the k8s app can expect to have a Pod
killed approximately every 3 hours.
**`kube-monkey/identifier`**: A unique identifier for the k8s apps. This is used to identify the pods
that belong to a k8s app as Pods inherit labels from their k8s app. So, if kube-monkey detects that app `foo` has enrolled to be a victim, kube-monkey will look for all pods that have the label `kube-monkey/identifier: foo` to determine which pods are candidates for killing. Recommendation is to set this value to be the same as the app's name.
**`kube-monkey/kill-mode`**: Default behavior is for kube-monkey to kill only ONE pod of your app. You can override this behavior by setting the value to:
Expand Down Expand Up @@ -52,7 +52,7 @@ spec:
labels:
kube-monkey/enabled: enabled
kube-monkey/identifier: monkey-victim
kube-monkey/mtbf: '2'
kube-monkey/mtbf: '2h'
kube-monkey/kill-mode: "fixed"
kube-monkey/kill-value: '1'
[... omitted ...]
Expand All @@ -70,7 +70,7 @@ metadata:
labels:
kube-monkey/enabled: enabled
kube-monkey/identifier: monkey-victim
kube-monkey/mtbf: '2'
kube-monkey/mtbf: '2h'
kube-monkey/kill-mode: "fixed"
kube-monkey/kill-value: '1'
spec:
Expand Down
32 changes: 32 additions & 0 deletions calendar/calendar.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,35 @@ func RandomTimeInRange(startHour int, endHour int, loc *time.Location) time.Time
rangeStart := time.Date(year, month, date, startHour, 0, 0, 0, loc)
return rangeStart.Add(offsetDuration)
}

func CustzRandomTimeInRange(mtbf string, startHour, endHour int, loc *time.Location) time.Time {
tmptimeDuration, err := time.ParseDuration(mtbf)
if err != nil {
glog.Errorf("error parsing customized mtbf %s: %v", mtbf, err)
return time.Now().Add(time.Duration(24*365*10) * time.Hour)
}
//time range should be twice of the input mean time between failure value
timeDuration := tmptimeDuration * 2
//compute random offset time
now := time.Now().In(loc)
mtbfEndTime := now.Add(timeDuration)
subSecond := int64(mtbfEndTime.Sub(now) / time.Second)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
randSecondOffset := r.Int63n(subSecond)
randCalTime := now.Add(time.Duration(randSecondOffset) * time.Second)

// compute randSecondOffset between start and end hour
year, month, date := now.Date()
todayEndTime := time.Date(year, month, date, endHour, 0, 0, 0, loc)
todayStartTime := time.Date(year, month, date, startHour, 0, 0, 0, loc)
if randCalTime.Before(todayEndTime) { // time offset before today's endHour
glog.V(1).Infof("CustzRandomTimeInRange calculate time %s", randCalTime)
return randCalTime
} else {
leftOffset := randSecondOffset - int64(todayEndTime.Sub(now)/time.Second)
offsetDay := leftOffset/(int64(endHour-startHour)*60*60) + 1
modOffsetSecond := leftOffset % (int64(endHour-startHour) * 60 * 60)
glog.V(1).Infof("CustzRandomTimeInRange calculate time %s", todayStartTime.Add(time.Duration(offsetDay*24)*time.Hour).Add(time.Duration(modOffsetSecond)*time.Second))
return todayStartTime.Add(time.Duration(offsetDay*24) * time.Hour).Add(time.Duration(modOffsetSecond) * time.Second)
}
}
20 changes: 20 additions & 0 deletions calendar/calendar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,23 @@ func TestRandomTimeInRange(t *testing.T) {

assert.Condition(t, scheduledTime)
}

func TestCustzRandomTimeInRange(t *testing.T) {
loc := time.UTC

monkey.Patch(time.Now, func() time.Time {
return time.Date(2018, 4, 16, 12, 0, 0, 0, time.UTC)
})
defer monkey.Unpatch(time.Now)

randomTime := CustzRandomTimeInRange("5h", 10, 12, loc)

scheduledTime := func() (success bool) {
if randomTime.Hour() >= 10 && randomTime.Hour() <= 12 {
success = true
}
return
}

assert.Condition(t, scheduledTime)
}
2 changes: 1 addition & 1 deletion chaos/chaosmock.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func (vm *victimMock) IsWhitelisted() bool {
}

func newVictimMock() *victimMock {
v := victims.New(KIND, NAME, NAMESPACE, IDENTIFIER, 1)
v := victims.New(KIND, NAME, NAMESPACE, IDENTIFIER, "1h")
return &victimMock{
VictimBase: *v,
}
Expand Down
11 changes: 5 additions & 6 deletions schedule/schedule.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,25 +72,24 @@ func New() (*Schedule, error) {
}

for _, victim := range victims {
killtime := CalculateKillTime()
killtime := CalculateKillTime(victim.Mtbf())

schedule.Add(chaos.New(killtime, victim))

if ShouldScheduleChaos(victim.Mtbf()) {
schedule.Add(chaos.New(killtime, victim))
}
}

return schedule, nil
}

func CalculateKillTime() time.Time {
func CalculateKillTime(mtbf string) time.Time {
loc := config.Timezone()
if config.DebugEnabled() && config.DebugScheduleImmediateKill() {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
// calculate a second-offset in the next minute
secOffset := r.Intn(60)
return time.Now().In(loc).Add(time.Duration(secOffset) * time.Second)
}
return calendar.RandomTimeInRange(config.StartHour(), config.EndHour(), loc)
return calendar.CustzRandomTimeInRange(mtbf, config.StartHour(), config.EndHour(), loc)
}

func ShouldScheduleChaos(mtbf int) bool {
Expand Down
4 changes: 2 additions & 2 deletions schedule/schedule_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func TestStringWithEntries(t *testing.T) {

func TestCalculateKillTimeRandom(t *testing.T) {
config.SetDefaults()
killtime := CalculateKillTime()
killtime := CalculateKillTime("1h")

scheduledTime := func() (success bool) {
if killtime.Hour() >= config.StartHour() && killtime.Hour() <= config.EndHour() {
Expand All @@ -83,7 +83,7 @@ func TestCalculateKillTimeNow(t *testing.T) {
config.SetDefaults()
viper.SetDefault(param.DebugEnabled, true)
viper.SetDefault(param.DebugScheduleImmediateKill, true)
killtime := CalculateKillTime()
killtime := CalculateKillTime("1h")

assert.Equal(t, killtime.Location(), config.Timezone())
assert.WithinDuration(t, killtime, time.Now(), time.Second*time.Duration(60))
Expand Down
16 changes: 6 additions & 10 deletions victims/factory/daemonsets/daemonsets.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package daemonsets

import (
"fmt"
"strconv"
"time"

"github.com/asobti/kube-monkey/config"
"github.com/asobti/kube-monkey/victims"
Expand Down Expand Up @@ -44,20 +44,16 @@ func identifier(kubekind *v1.DaemonSet) (string, error) {

// Read the mean-time-between-failures value defined by the DaemonSet
// in the label defined by config.MtbfLabelKey
func meanTimeBetweenFailures(kubekind *v1.DaemonSet) (int, error) {
func meanTimeBetweenFailures(kubekind *v1.DaemonSet) (string, error) {
mtbf, ok := kubekind.Labels[config.MtbfLabelKey]
if !ok {
return -1, fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey)
return "", fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey)
}

mtbfInt, err := strconv.Atoi(mtbf)
_, err := time.ParseDuration(mtbf)
if err != nil {
return -1, err
return "", fmt.Errorf("error parsing mtbf %s: %v", mtbf, err)
}

if !(mtbfInt > 0) {
return -1, fmt.Errorf("Invalid value for label %s: %d", config.MtbfLabelKey, mtbfInt)
}

return mtbfInt, nil
return mtbf, nil
}
18 changes: 4 additions & 14 deletions victims/factory/daemonsets/daemonsets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestNew(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: IDENTIFIER,
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
},
)
ds, err := New(&v1ds)
Expand All @@ -42,14 +42,14 @@ func TestNew(t *testing.T) {
assert.Equal(t, NAME, ds.Name())
assert.Equal(t, NAMESPACE, ds.Namespace())
assert.Equal(t, IDENTIFIER, ds.Identifier())
assert.Equal(t, 1, ds.Mtbf())
assert.Equal(t, "1h", ds.Mtbf())
}

func TestInvalidIdentifier(t *testing.T) {
v1ds := newDaemonSet(
NAME,
map[string]string{
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
},
)
_, err := New(&v1ds)
Expand Down Expand Up @@ -77,16 +77,6 @@ func TestInvalidMtbf(t *testing.T) {
)
_, err = New(&v1ds)

assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a Int type")
assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a time.Duration type")

v1ds = newDaemonSet(
NAME,
map[string]string{
config.IdentLabelKey: IDENTIFIER,
config.MtbfLabelKey: "0",
},
)
_, err = New(&v1ds)

assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label is lower than 1")
}
10 changes: 5 additions & 5 deletions victims/factory/daemonsets/eligible_daemonsets_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func TestEligibleDaemonSets(t *testing.T) {
NAME,
map[string]string{
"kube-monkey/identifier": "1",
"kube-monkey/mtbf": "1",
"kube-monkey/mtbf": "1h",
},
)

Expand All @@ -29,7 +29,7 @@ func TestIsEnrolled(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: "1",
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
config.EnabledLabelKey: config.EnabledLabelValue,
},
)
Expand All @@ -48,7 +48,7 @@ func TestIsNotEnrolled(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: "1",
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
config.EnabledLabelKey: "x",
},
)
Expand All @@ -65,7 +65,7 @@ func TestIsNotEnrolled(t *testing.T) {
func TestKillType(t *testing.T) {

ident := "1"
mtbf := "1"
mtbf := "1h"
killMode := "kill-mode"

v1ds := newDaemonSet(
Expand Down Expand Up @@ -103,7 +103,7 @@ func TestKillType(t *testing.T) {
func TestKillValue(t *testing.T) {

ident := "1"
mtbf := "1"
mtbf := "1h"
killValue := "0"

v1ds := newDaemonSet(
Expand Down
17 changes: 6 additions & 11 deletions victims/factory/deployments/deployments.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@ package deployments

import (
"fmt"
"strconv"

"github.com/asobti/kube-monkey/config"
"github.com/asobti/kube-monkey/victims"
"time"

"k8s.io/api/apps/v1"
)
Expand Down Expand Up @@ -44,20 +43,16 @@ func identifier(kubekind *v1.Deployment) (string, error) {

// Read the mean-time-between-failures value defined by the Deployment
// in the label defined by config.MtbfLabelKey
func meanTimeBetweenFailures(kubekind *v1.Deployment) (int, error) {
func meanTimeBetweenFailures(kubekind *v1.Deployment) (string, error) {
mtbf, ok := kubekind.Labels[config.MtbfLabelKey]
if !ok {
return -1, fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey)
return "", fmt.Errorf("%T %s does not have %s label", kubekind, kubekind.Name, config.MtbfLabelKey)
}

mtbfInt, err := strconv.Atoi(mtbf)
_, err := time.ParseDuration(mtbf)
if err != nil {
return -1, err
}

if !(mtbfInt > 0) {
return -1, fmt.Errorf("Invalid value for label %s: %d", config.MtbfLabelKey, mtbfInt)
return "", fmt.Errorf("error parsing mtbf %s: %v", mtbf, err)
}

return mtbfInt, nil
return mtbf, nil
}
19 changes: 4 additions & 15 deletions victims/factory/deployments/deployments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestNew(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: IDENTIFIER,
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
},
)
depl, err := New(&v1depl)
Expand All @@ -42,14 +42,14 @@ func TestNew(t *testing.T) {
assert.Equal(t, NAME, depl.Name())
assert.Equal(t, NAMESPACE, depl.Namespace())
assert.Equal(t, IDENTIFIER, depl.Identifier())
assert.Equal(t, 1, depl.Mtbf())
assert.Equal(t, "1h", depl.Mtbf())
}

func TestInvalidIdentifier(t *testing.T) {
v1depl := newDeployment(
NAME,
map[string]string{
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
},
)
_, err := New(&v1depl)
Expand Down Expand Up @@ -77,16 +77,5 @@ func TestInvalidMtbf(t *testing.T) {
)
_, err = New(&v1depl)

assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a Int type")

v1depl = newDeployment(
NAME,
map[string]string{
config.IdentLabelKey: IDENTIFIER,
config.MtbfLabelKey: "0",
},
)
_, err = New(&v1depl)

assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label is lower than 1")
assert.Errorf(t, err, "Expected an error if "+config.MtbfLabelKey+" label can't be converted a time.Duration type")
}
10 changes: 5 additions & 5 deletions victims/factory/deployments/eligible_deployments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ func TestEligibleDeployments(t *testing.T) {
NAME,
map[string]string{
"kube-monkey/identifier": "1",
"kube-monkey/mtbf": "1",
"kube-monkey/mtbf": "1h",
},
)

Expand All @@ -29,7 +29,7 @@ func TestIsEnrolled(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: "1",
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
config.EnabledLabelKey: config.EnabledLabelValue,
},
)
Expand All @@ -48,7 +48,7 @@ func TestIsNotEnrolled(t *testing.T) {
NAME,
map[string]string{
config.IdentLabelKey: "1",
config.MtbfLabelKey: "1",
config.MtbfLabelKey: "1h",
config.EnabledLabelKey: "x",
},
)
Expand All @@ -65,7 +65,7 @@ func TestIsNotEnrolled(t *testing.T) {
func TestKillType(t *testing.T) {

ident := "1"
mtbf := "1"
mtbf := "1h"
killMode := "kill-mode"

v1depl := newDeployment(
Expand Down Expand Up @@ -103,7 +103,7 @@ func TestKillType(t *testing.T) {
func TestKillValue(t *testing.T) {

ident := "1"
mtbf := "1"
mtbf := "1h"
killValue := "0"

v1depl := newDeployment(
Expand Down
Loading

0 comments on commit 175ec88

Please sign in to comment.