jayjay
/
k9s
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

enable finer controls for cluster health alerts

mine
derailed 2020-03-04 17:10:08 -07:00
parent b2bb15bfd7
commit 044f7aa663
8 changed files with 235 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
internal/config/config_test.go
View File

@ -298,9 +298,16 @@ var expectedConfig = `k9s:
view:
active: ctx
thresholds:
cpu: 80
memory: 80
disk: 80
cpu:
- 90
- 80
- 75
- 70
memory:
- 90
- 80
- 75
- 70
`
var resetConfig = `k9s:
@ -321,7 +328,14 @@ var resetConfig = `k9s:
view:
active: po
thresholds:
cpu: 80
memory: 80
disk: 80
cpu:
- 90
- 80
- 75
- 70
memory:
- 90
- 80
- 75
- 70
`

6
internal/config/k9s.go
View File

@ -20,7 +20,7 @@ type K9s struct {
CurrentCluster string `yaml:"currentCluster"`
FullScreenLogs bool `yaml:"fullScreenLogs"`
Clusters map[string]*Cluster `yaml:"clusters,omitempty"`
Thresholds *Threshold `yaml:"thresholds"`
Thresholds Threshold `yaml:"thresholds"`
manualRefreshRate int
manualHeadless *bool
manualReadOnly *bool
@ -35,7 +35,7 @@ func NewK9s() *K9s {
LogBufferSize: defaultLogBufferSize,
LogRequestSize: defaultLogRequestSize,
Clusters: make(map[string]*Cluster),
Thresholds: newThreshold(),
Thresholds: NewThreshold(),
}
}
@ -141,7 +141,7 @@ func (k *K9s) Validate(c client.Connection, ks KubeSettings) {
k.checkClusters(ks)
if k.Thresholds == nil {
k.Thresholds = newThreshold()
k.Thresholds = NewThreshold()
}
k.Thresholds.Validate(c, ks)

126
internal/config/threshold.go
View File

@ -1,54 +1,118 @@
package config
import (
"strings"
"github.com/derailed/k9s/internal/client"
"github.com/derailed/k9s/internal/render"
)
const (
defaultCPU = 80
defaultMEM = 80
defaultDisk = 70
DefCon1 DefConLevel = iota + 1
DefCon2
DefCon3
DefCon4
DefCon5
)
// Threshold tracks threshold to alert user when excided.
type Threshold struct {
CPU int `yaml:"cpu"`
Memory int `yaml:"memory"`
Disk int `yaml:"disk"`
type DefConLevel int
// DefCon tracks a resource alert level.
type DefCon [4]int
func newDefCon() DefCon {
return DefCon{90, 80, 75, 70}
}
func newThreshold() *Threshold {
return &Threshold{
CPU: defaultCPU,
Memory: defaultMEM,
Disk: defaultMEM,
func (d DefCon) validate() {
dc := newDefCon()
for i := range d {
if !d.isValidRange(d[i]) {
d[i] = dc[i]
}
}
}
func (d DefCon) String() string {
ss := make([]string, len(d))
for i := 0; i < len(d); i++ {
ss[i] = render.PrintPerc(d[i])
}
return strings.Join(ss, "|")
}
func (d DefCon) isValidRange(v int) bool {
if v == 0 || v > 100 {
return false
}
return true
}
// Threshold tracks threshold to alert user when excided.
type Threshold map[string]DefCon
func NewThreshold() Threshold {
return Threshold{
"cpu": newDefCon(),
"memory": newDefCon(),
}
}
// Validate a namespace is setup correctly
func (t *Threshold) Validate(c client.Connection, ks KubeSettings) {
if t.CPU == 0 || t.CPU > 100 {
t.CPU = defaultCPU
}
if t.Memory == 0 || t.Memory > 100 {
t.Memory = defaultMEM
}
if t.Disk == 0 || t.Disk > 100 {
t.Disk = defaultDisk
func (t Threshold) Validate(c client.Connection, ks KubeSettings) {
for _, k := range []string{"cpu", "memory"} {
v, ok := t[k]
if !ok {
t[k] = newDefCon()
} else {
v.validate()
}
}
}
// ExceedsCPUPerc returns true if current metrics exceeds threshold or false otherwise.
func (t *Threshold) ExceedsCPUPerc(p int) bool {
return p >= t.CPU
// DefConFor returns a defcon level for the current state.
func (t Threshold) DefConFor(k string, v int) DefConLevel {
dc, ok := t[k]
if !ok || v < 0 || v > 100 {
return DefCon5
}
for i, l := range dc {
if v >= l {
return dcLevelFor(i)
}
}
return DefCon5
}
// ExceedsMemoryPerc returns true if current metrics exceeds threshold or false otherwise.
func (t *Threshold) ExceedsMemoryPerc(p int) bool {
return p >= t.Memory
func (t *Threshold) DefConColorFor(k string, v int) string {
switch t.DefConFor(k, v) {
case DefCon1:
return "red"
case DefCon2:
return "orangered"
case DefCon3:
return "orange"
default:
return "green"
}
}
// ExceedsDiskPerc returns true if current metrics exceeds threshold or false otherwise.
func (t *Threshold) ExceedsDiskPerc(p int) bool {
return p >= t.Disk
// ----------------------------------------------------------------------------
// Helpers...
func dcLevelFor(l int) DefConLevel {
switch l {
case 0:
return DefCon1
case 1:
return DefCon2
case 2:
return DefCon3
case 3:
return DefCon4
default:
return DefCon5
}
}

56
internal/config/threshold_test.go Normal file
View File

@ -0,0 +1,56 @@
package config_test
import (
"testing"
"github.com/derailed/k9s/internal/config"
"github.com/stretchr/testify/assert"
)
func TestDefConFor(t *testing.T) {
uu := map[string]struct {
k string
v int
e config.DefConLevel
}{
"normal": {
k: "cpu",
v: 0,
e: config.DefCon5,
},
"4": {
k: "cpu",
v: 71,
e: config.DefCon4,
},
"3": {
k: "cpu",
v: 75,
e: config.DefCon3,
},
"2": {
k: "cpu",
v: 80,
e: config.DefCon2,
},
"1": {
k: "cpu",
v: 100,
e: config.DefCon1,
},
"over": {
k: "cpu",
v: 150,
e: config.DefCon5,
},
}
o := config.NewThreshold()
for k := range uu {
u := uu[k]
t.Run(k, func(t *testing.T) {
assert.Equal(t, u.e, o.DefConFor(u.k, u.v))
})
}
}

33
internal/tchart/gauge.go
View File

@ -58,6 +58,13 @@ func (g *Gauge) Add(m Metric) {
g.data = m
}
type number struct {
ok bool
val int64
str string
delta delta
}
// Draw draws the primitive.
func (g *Gauge) Draw(sc tcell.Screen) {
g.Component.Draw(sc)
@ -83,10 +90,10 @@ func (g *Gauge) Draw(sc tcell.Screen) {
s1C, s2C := g.colorForSeries()
d1, d2 := fmt.Sprintf(fmat, g.data.S1), fmt.Sprintf(fmat, g.data.S2)
o.X -= len(d1) * 3
g.drawNum(sc, true, o, g.data.S1, g.deltaOk, d1, style.Foreground(s1C).Dim(false))
g.drawNum(sc, o, number{ok: true, val: g.data.S1, delta: g.deltaOk, str: d1}, style.Foreground(s1C).Dim(false))
o.X = mid.X + 1
g.drawNum(sc, false, o, g.data.S2, g.deltaS2, d2, style.Foreground(s2C).Dim(false))
g.drawNum(sc, o, number{ok: false, val: g.data.S2, delta: g.deltaS2, str: d2}, style.Foreground(s2C).Dim(false))
if rect.Dx() > 0 && rect.Dy() > 0 && g.legend != "" {
legend := g.legend
@ -97,29 +104,29 @@ func (g *Gauge) Draw(sc tcell.Screen) {
}
}
func (g *Gauge) drawNum(sc tcell.Screen, ok bool, o image.Point, n int64, dn delta, ns string, style tcell.Style) {
func (g *Gauge) drawNum(sc tcell.Screen, o image.Point, n number, style tcell.Style) {
c1, _ := g.colorForSeries()
if ok {
if n.ok {
style = style.Foreground(c1)
printDelta(sc, dn, o, style)
printDelta(sc, n.delta, o, style)
}
dm, significant := NewDotMatrix(), n == 0
if n == 0 {
dm, significant := NewDotMatrix(), n.val == 0
if significant {
style = g.dimmed
}
for i := 0; i < len(ns); i++ {
if ns[i] == '0' && !significant {
g.drawDial(sc, dm.Print(int(ns[i]-48)), o, g.dimmed)
for i := 0; i < len(n.str); i++ {
if n.str[i] == '0' && !significant {
g.drawDial(sc, dm.Print(int(n.str[i]-48)), o, g.dimmed)
} else {
significant = true
g.drawDial(sc, dm.Print(int(ns[i]-48)), o, style)
g.drawDial(sc, dm.Print(int(n.str[i]-48)), o, style)
}
o.X += 3
}
if !ok {
if !n.ok {
o.X++
printDelta(sc, dn, o, style)
printDelta(sc, n.delta, o, style)
}
}

47
internal/view/cluster_info.go
View File

@ -1,6 +1,8 @@
package view
import (
"fmt"
"github.com/derailed/k9s/internal/config"
"github.com/derailed/k9s/internal/model"
"github.com/derailed/k9s/internal/render"
@ -93,23 +95,33 @@ func (c *ClusterInfo) ClusterInfoChanged(prev, curr model.ClusterMeta) {
if c.app.Conn().HasMetrics() {
row = c.setCell(row, ui.AsPercDelta(prev.Cpu, curr.Cpu))
_ = c.setCell(row, ui.AsPercDelta(prev.Mem, curr.Mem))
var set bool
if c.app.Config.K9s.Thresholds.ExceedsCPUPerc(curr.Cpu) {
c.app.Status(model.FlashErr, "CPU on fire!")
set = true
}
if c.app.Config.K9s.Thresholds.ExceedsMemoryPerc(curr.Mem) {
c.app.Status(model.FlashErr, "Memory on fire!")
set = true
}
if !set {
c.app.ClearStatus(true)
}
c.setDefCon(curr.Cpu, curr.Mem)
}
c.updateStyle()
})
}
const defconFmt = "Cluster <%s> at DEFCON %d"
func (c *ClusterInfo) setDefCon(cpu, mem int) {
var set bool
dc := c.app.Config.K9s.Thresholds.DefConFor("cpu", cpu)
if dc < config.DefCon5 {
l := flashFromDefCon(dc)
c.app.Status(l, fmt.Sprintf(defconFmt, "cpu", int(dc)))
set = true
}
dc = c.app.Config.K9s.Thresholds.DefConFor("memory", mem)
if dc < config.DefCon5 {
l := flashFromDefCon(dc)
c.app.Status(l, fmt.Sprintf(defconFmt, "mem", int(dc)))
set = true
}
if !set {
c.app.ClearStatus(true)
}
}
func (c *ClusterInfo) updateStyle() {
for row := 0; row < c.GetRowCount(); row++ {
c.GetCell(row, 0).SetTextColor(c.styles.K9s.Info.FgColor.Color())
@ -118,3 +130,14 @@ func (c *ClusterInfo) updateStyle() {
c.GetCell(row, 1).SetStyle(s.Bold(true).Foreground(c.styles.K9s.Info.SectionColor.Color()))
}
}
func flashFromDefCon(l config.DefConLevel) model.FlashLevel {
switch l {
case config.DefCon1:
return model.FlashErr
case config.DefCon2, config.DefCon3:
return model.FlashWarn
default:
return model.FlashInfo
}
}

22
internal/view/pulse.go
View File

@ -128,11 +128,9 @@ func (p *Pulse) StylesChanged(s *config.Styles) {
}
const (
genFmat = " %s([%s::]%d[white::]:[%s::b]%d[-::])"
cpuFmt = " %s [%s::b]%s[white::-]::[gray::]%s ([%s::]%sm[white::]/[%s::]%sm[-::])"
memFmt = " %s [%s::b]%s[white::-]::[gray::]%s ([%s::]%sMi[white::]/[%s::]%sMi[-::])"
okColor = "palegreen"
errColor = "orangered"
genFmat = " %s([%s::]%d[white::]:[%s::b]%d[-::])"
cpuFmt = " %s [%s::b]%s[white::-]([%s::]%sm[white::]/[%s::]%sm[-::])"
memFmt = " %s [%s::b]%s[white::-]([%s::]%sMi[white::]/[%s::]%sMi[-::])"
)
// PulseChanged notifies the model data changed.
@ -159,15 +157,10 @@ func (p *Pulse) PulseChanged(c *health.Check) {
switch c.GVR {
case "cpu":
perc := client.ToPercentage(c.Tally(health.S1), c.Tally(health.S2))
color := okColor
if p.app.Config.K9s.Thresholds.ExceedsCPUPerc(perc) {
color = errColor
}
v.SetLegend(fmt.Sprintf(cpuFmt,
strings.Title(gvr.R()),
color,
p.app.Config.K9s.Thresholds.DefConColorFor("cpu", perc),
render.PrintPerc(perc),
render.PrintPerc(p.app.Config.K9s.Thresholds.CPU),
nn[0],
render.AsThousands(c.Tally(health.S1)),
nn[1],
@ -175,15 +168,10 @@ func (p *Pulse) PulseChanged(c *health.Check) {
))
case "mem":
perc := client.ToPercentage(c.Tally(health.S1), c.Tally(health.S2))
color := okColor
if p.app.Config.K9s.Thresholds.ExceedsMemoryPerc(perc) {
color = errColor
}
v.SetLegend(fmt.Sprintf(memFmt,
strings.Title(gvr.R()),
color,
p.app.Config.K9s.Thresholds.DefConColorFor("memory", perc),
render.PrintPerc(perc),
render.PrintPerc(p.app.Config.K9s.Thresholds.Memory),
nn[0],
render.AsThousands(c.Tally(health.S1)),
nn[1],

2
internal/view/restart_extender.go
View File

@ -38,7 +38,7 @@ func (r *RestartExtender) restartCmd(evt *tcell.EventKey) *tcell.EventKey {
r.Stop()
defer r.Start()
msg := fmt.Sprintf("Restart deployment %s?" + paths[0])
msg := fmt.Sprintf("Restart deployment %s?", paths[0])
if len(paths) > 1 {
msg = fmt.Sprintf("Restart %d deployments?", len(paths))
}