From 044f7aa6631b620a40f8b95930a1376110986430 Mon Sep 17 00:00:00 2001 From: derailed Date: Wed, 4 Mar 2020 17:10:08 -0700 Subject: [PATCH] enable finer controls for cluster health alerts --- internal/config/config_test.go | 26 ++++-- internal/config/k9s.go | 6 +- internal/config/threshold.go | 126 ++++++++++++++++++++++-------- internal/config/threshold_test.go | 56 +++++++++++++ internal/tchart/gauge.go | 33 +++++--- internal/view/cluster_info.go | 47 ++++++++--- internal/view/pulse.go | 22 ++---- internal/view/restart_extender.go | 2 +- 8 files changed, 235 insertions(+), 83 deletions(-) create mode 100644 internal/config/threshold_test.go diff --git a/internal/config/config_test.go b/internal/config/config_test.go index cd34777f..79351b8b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -298,9 +298,16 @@ var expectedConfig = `k9s: view: active: ctx thresholds: - cpu: 80 - memory: 80 - disk: 80 + cpu: + - 90 + - 80 + - 75 + - 70 + memory: + - 90 + - 80 + - 75 + - 70 ` var resetConfig = `k9s: @@ -321,7 +328,14 @@ var resetConfig = `k9s: view: active: po thresholds: - cpu: 80 - memory: 80 - disk: 80 + cpu: + - 90 + - 80 + - 75 + - 70 + memory: + - 90 + - 80 + - 75 + - 70 ` diff --git a/internal/config/k9s.go b/internal/config/k9s.go index 6f416920..0e6cfdc4 100644 --- a/internal/config/k9s.go +++ b/internal/config/k9s.go @@ -20,7 +20,7 @@ type K9s struct { CurrentCluster string `yaml:"currentCluster"` FullScreenLogs bool `yaml:"fullScreenLogs"` Clusters map[string]*Cluster `yaml:"clusters,omitempty"` - Thresholds *Threshold `yaml:"thresholds"` + Thresholds Threshold `yaml:"thresholds"` manualRefreshRate int manualHeadless *bool manualReadOnly *bool @@ -35,7 +35,7 @@ func NewK9s() *K9s { LogBufferSize: defaultLogBufferSize, LogRequestSize: defaultLogRequestSize, Clusters: make(map[string]*Cluster), - Thresholds: newThreshold(), + Thresholds: NewThreshold(), } } @@ -141,7 +141,7 @@ func (k *K9s) Validate(c client.Connection, ks KubeSettings) { k.checkClusters(ks) if k.Thresholds == nil { - k.Thresholds = newThreshold() + k.Thresholds = NewThreshold() } k.Thresholds.Validate(c, ks) diff --git a/internal/config/threshold.go b/internal/config/threshold.go index 445603ab..6b76154e 100644 --- a/internal/config/threshold.go +++ b/internal/config/threshold.go @@ -1,54 +1,118 @@ package config import ( + "strings" + "github.com/derailed/k9s/internal/client" + "github.com/derailed/k9s/internal/render" ) const ( - defaultCPU = 80 - defaultMEM = 80 - defaultDisk = 70 + DefCon1 DefConLevel = iota + 1 + DefCon2 + DefCon3 + DefCon4 + DefCon5 ) -// Threshold tracks threshold to alert user when excided. -type Threshold struct { - CPU int `yaml:"cpu"` - Memory int `yaml:"memory"` - Disk int `yaml:"disk"` +type DefConLevel int + +// DefCon tracks a resource alert level. +type DefCon [4]int + +func newDefCon() DefCon { + return DefCon{90, 80, 75, 70} } -func newThreshold() *Threshold { - return &Threshold{ - CPU: defaultCPU, - Memory: defaultMEM, - Disk: defaultMEM, +func (d DefCon) validate() { + dc := newDefCon() + for i := range d { + if !d.isValidRange(d[i]) { + d[i] = dc[i] + } + } +} + +func (d DefCon) String() string { + ss := make([]string, len(d)) + for i := 0; i < len(d); i++ { + ss[i] = render.PrintPerc(d[i]) + } + return strings.Join(ss, "|") +} + +func (d DefCon) isValidRange(v int) bool { + if v == 0 || v > 100 { + return false + } + + return true +} + +// Threshold tracks threshold to alert user when excided. +type Threshold map[string]DefCon + +func NewThreshold() Threshold { + return Threshold{ + "cpu": newDefCon(), + "memory": newDefCon(), } } // Validate a namespace is setup correctly -func (t *Threshold) Validate(c client.Connection, ks KubeSettings) { - if t.CPU == 0 || t.CPU > 100 { - t.CPU = defaultCPU - } - if t.Memory == 0 || t.Memory > 100 { - t.Memory = defaultMEM - } - if t.Disk == 0 || t.Disk > 100 { - t.Disk = defaultDisk +func (t Threshold) Validate(c client.Connection, ks KubeSettings) { + for _, k := range []string{"cpu", "memory"} { + v, ok := t[k] + if !ok { + t[k] = newDefCon() + } else { + v.validate() + } } } -// ExceedsCPUPerc returns true if current metrics exceeds threshold or false otherwise. -func (t *Threshold) ExceedsCPUPerc(p int) bool { - return p >= t.CPU +// DefConFor returns a defcon level for the current state. +func (t Threshold) DefConFor(k string, v int) DefConLevel { + dc, ok := t[k] + if !ok || v < 0 || v > 100 { + return DefCon5 + } + for i, l := range dc { + if v >= l { + return dcLevelFor(i) + } + } + + return DefCon5 } -// ExceedsMemoryPerc returns true if current metrics exceeds threshold or false otherwise. -func (t *Threshold) ExceedsMemoryPerc(p int) bool { - return p >= t.Memory +func (t *Threshold) DefConColorFor(k string, v int) string { + switch t.DefConFor(k, v) { + case DefCon1: + return "red" + case DefCon2: + return "orangered" + case DefCon3: + return "orange" + default: + return "green" + } } -// ExceedsDiskPerc returns true if current metrics exceeds threshold or false otherwise. -func (t *Threshold) ExceedsDiskPerc(p int) bool { - return p >= t.Disk +// ---------------------------------------------------------------------------- +// Helpers... + +func dcLevelFor(l int) DefConLevel { + switch l { + case 0: + return DefCon1 + case 1: + return DefCon2 + case 2: + return DefCon3 + case 3: + return DefCon4 + default: + return DefCon5 + } } diff --git a/internal/config/threshold_test.go b/internal/config/threshold_test.go new file mode 100644 index 00000000..a5d633b8 --- /dev/null +++ b/internal/config/threshold_test.go @@ -0,0 +1,56 @@ +package config_test + +import ( + "testing" + + "github.com/derailed/k9s/internal/config" + "github.com/stretchr/testify/assert" +) + +func TestDefConFor(t *testing.T) { + uu := map[string]struct { + k string + v int + e config.DefConLevel + }{ + "normal": { + k: "cpu", + v: 0, + + e: config.DefCon5, + }, + "4": { + k: "cpu", + v: 71, + e: config.DefCon4, + }, + "3": { + k: "cpu", + v: 75, + e: config.DefCon3, + }, + "2": { + k: "cpu", + v: 80, + e: config.DefCon2, + }, + "1": { + k: "cpu", + v: 100, + e: config.DefCon1, + }, + "over": { + k: "cpu", + v: 150, + e: config.DefCon5, + }, + } + + o := config.NewThreshold() + for k := range uu { + u := uu[k] + t.Run(k, func(t *testing.T) { + assert.Equal(t, u.e, o.DefConFor(u.k, u.v)) + }) + } +} diff --git a/internal/tchart/gauge.go b/internal/tchart/gauge.go index 9b42fb3c..07fcb81a 100644 --- a/internal/tchart/gauge.go +++ b/internal/tchart/gauge.go @@ -58,6 +58,13 @@ func (g *Gauge) Add(m Metric) { g.data = m } +type number struct { + ok bool + val int64 + str string + delta delta +} + // Draw draws the primitive. func (g *Gauge) Draw(sc tcell.Screen) { g.Component.Draw(sc) @@ -83,10 +90,10 @@ func (g *Gauge) Draw(sc tcell.Screen) { s1C, s2C := g.colorForSeries() d1, d2 := fmt.Sprintf(fmat, g.data.S1), fmt.Sprintf(fmat, g.data.S2) o.X -= len(d1) * 3 - g.drawNum(sc, true, o, g.data.S1, g.deltaOk, d1, style.Foreground(s1C).Dim(false)) + g.drawNum(sc, o, number{ok: true, val: g.data.S1, delta: g.deltaOk, str: d1}, style.Foreground(s1C).Dim(false)) o.X = mid.X + 1 - g.drawNum(sc, false, o, g.data.S2, g.deltaS2, d2, style.Foreground(s2C).Dim(false)) + g.drawNum(sc, o, number{ok: false, val: g.data.S2, delta: g.deltaS2, str: d2}, style.Foreground(s2C).Dim(false)) if rect.Dx() > 0 && rect.Dy() > 0 && g.legend != "" { legend := g.legend @@ -97,29 +104,29 @@ func (g *Gauge) Draw(sc tcell.Screen) { } } -func (g *Gauge) drawNum(sc tcell.Screen, ok bool, o image.Point, n int64, dn delta, ns string, style tcell.Style) { +func (g *Gauge) drawNum(sc tcell.Screen, o image.Point, n number, style tcell.Style) { c1, _ := g.colorForSeries() - if ok { + if n.ok { style = style.Foreground(c1) - printDelta(sc, dn, o, style) + printDelta(sc, n.delta, o, style) } - dm, significant := NewDotMatrix(), n == 0 - if n == 0 { + dm, significant := NewDotMatrix(), n.val == 0 + if significant { style = g.dimmed } - for i := 0; i < len(ns); i++ { - if ns[i] == '0' && !significant { - g.drawDial(sc, dm.Print(int(ns[i]-48)), o, g.dimmed) + for i := 0; i < len(n.str); i++ { + if n.str[i] == '0' && !significant { + g.drawDial(sc, dm.Print(int(n.str[i]-48)), o, g.dimmed) } else { significant = true - g.drawDial(sc, dm.Print(int(ns[i]-48)), o, style) + g.drawDial(sc, dm.Print(int(n.str[i]-48)), o, style) } o.X += 3 } - if !ok { + if !n.ok { o.X++ - printDelta(sc, dn, o, style) + printDelta(sc, n.delta, o, style) } } diff --git a/internal/view/cluster_info.go b/internal/view/cluster_info.go index b2149b1f..d81d9372 100644 --- a/internal/view/cluster_info.go +++ b/internal/view/cluster_info.go @@ -1,6 +1,8 @@ package view import ( + "fmt" + "github.com/derailed/k9s/internal/config" "github.com/derailed/k9s/internal/model" "github.com/derailed/k9s/internal/render" @@ -93,23 +95,33 @@ func (c *ClusterInfo) ClusterInfoChanged(prev, curr model.ClusterMeta) { if c.app.Conn().HasMetrics() { row = c.setCell(row, ui.AsPercDelta(prev.Cpu, curr.Cpu)) _ = c.setCell(row, ui.AsPercDelta(prev.Mem, curr.Mem)) - var set bool - if c.app.Config.K9s.Thresholds.ExceedsCPUPerc(curr.Cpu) { - c.app.Status(model.FlashErr, "CPU on fire!") - set = true - } - if c.app.Config.K9s.Thresholds.ExceedsMemoryPerc(curr.Mem) { - c.app.Status(model.FlashErr, "Memory on fire!") - set = true - } - if !set { - c.app.ClearStatus(true) - } + c.setDefCon(curr.Cpu, curr.Mem) } c.updateStyle() }) } +const defconFmt = "Cluster <%s> at DEFCON %d" + +func (c *ClusterInfo) setDefCon(cpu, mem int) { + var set bool + dc := c.app.Config.K9s.Thresholds.DefConFor("cpu", cpu) + if dc < config.DefCon5 { + l := flashFromDefCon(dc) + c.app.Status(l, fmt.Sprintf(defconFmt, "cpu", int(dc))) + set = true + } + dc = c.app.Config.K9s.Thresholds.DefConFor("memory", mem) + if dc < config.DefCon5 { + l := flashFromDefCon(dc) + c.app.Status(l, fmt.Sprintf(defconFmt, "mem", int(dc))) + set = true + } + if !set { + c.app.ClearStatus(true) + } +} + func (c *ClusterInfo) updateStyle() { for row := 0; row < c.GetRowCount(); row++ { c.GetCell(row, 0).SetTextColor(c.styles.K9s.Info.FgColor.Color()) @@ -118,3 +130,14 @@ func (c *ClusterInfo) updateStyle() { c.GetCell(row, 1).SetStyle(s.Bold(true).Foreground(c.styles.K9s.Info.SectionColor.Color())) } } + +func flashFromDefCon(l config.DefConLevel) model.FlashLevel { + switch l { + case config.DefCon1: + return model.FlashErr + case config.DefCon2, config.DefCon3: + return model.FlashWarn + default: + return model.FlashInfo + } +} diff --git a/internal/view/pulse.go b/internal/view/pulse.go index 11df4398..790c3005 100644 --- a/internal/view/pulse.go +++ b/internal/view/pulse.go @@ -128,11 +128,9 @@ func (p *Pulse) StylesChanged(s *config.Styles) { } const ( - genFmat = " %s([%s::]%d[white::]:[%s::b]%d[-::])" - cpuFmt = " %s [%s::b]%s[white::-]::[gray::]%s ([%s::]%sm[white::]/[%s::]%sm[-::])" - memFmt = " %s [%s::b]%s[white::-]::[gray::]%s ([%s::]%sMi[white::]/[%s::]%sMi[-::])" - okColor = "palegreen" - errColor = "orangered" + genFmat = " %s([%s::]%d[white::]:[%s::b]%d[-::])" + cpuFmt = " %s [%s::b]%s[white::-]([%s::]%sm[white::]/[%s::]%sm[-::])" + memFmt = " %s [%s::b]%s[white::-]([%s::]%sMi[white::]/[%s::]%sMi[-::])" ) // PulseChanged notifies the model data changed. @@ -159,15 +157,10 @@ func (p *Pulse) PulseChanged(c *health.Check) { switch c.GVR { case "cpu": perc := client.ToPercentage(c.Tally(health.S1), c.Tally(health.S2)) - color := okColor - if p.app.Config.K9s.Thresholds.ExceedsCPUPerc(perc) { - color = errColor - } v.SetLegend(fmt.Sprintf(cpuFmt, strings.Title(gvr.R()), - color, + p.app.Config.K9s.Thresholds.DefConColorFor("cpu", perc), render.PrintPerc(perc), - render.PrintPerc(p.app.Config.K9s.Thresholds.CPU), nn[0], render.AsThousands(c.Tally(health.S1)), nn[1], @@ -175,15 +168,10 @@ func (p *Pulse) PulseChanged(c *health.Check) { )) case "mem": perc := client.ToPercentage(c.Tally(health.S1), c.Tally(health.S2)) - color := okColor - if p.app.Config.K9s.Thresholds.ExceedsMemoryPerc(perc) { - color = errColor - } v.SetLegend(fmt.Sprintf(memFmt, strings.Title(gvr.R()), - color, + p.app.Config.K9s.Thresholds.DefConColorFor("memory", perc), render.PrintPerc(perc), - render.PrintPerc(p.app.Config.K9s.Thresholds.Memory), nn[0], render.AsThousands(c.Tally(health.S1)), nn[1], diff --git a/internal/view/restart_extender.go b/internal/view/restart_extender.go index 03883179..2f891908 100644 --- a/internal/view/restart_extender.go +++ b/internal/view/restart_extender.go @@ -38,7 +38,7 @@ func (r *RestartExtender) restartCmd(evt *tcell.EventKey) *tcell.EventKey { r.Stop() defer r.Start() - msg := fmt.Sprintf("Restart deployment %s?" + paths[0]) + msg := fmt.Sprintf("Restart deployment %s?", paths[0]) if len(paths) > 1 { msg = fmt.Sprintf("Restart %d deployments?", len(paths)) }