fix: consider readinessGates + ready condition in diagnose (#3556)
* Consider readinessGates + ready condition in diagnose Now that readiness gates are supported, those should be considered when displaying a pod healthiness Additionally consider the pod ready condition status. It should match the && of the containers' ready condition and readiness gates but I've actually observed that in some cases, it can be false with the containers reporting ready true. * Update pod.go * Update pod.go fix lint * Update pod.go lint * add tests for diagnose and readinessGateStats * lint errormine
jfremy-openai
GitHub
parent
2daec83d60
commit
41acad343b
|
|
@ -165,6 +165,8 @@ func (p *Pod) defaultRow(pwm *PodWithMetrics, row *model1.Row) error {
|
|||
iReady, iTerminated, iRestarts := p.initContainerStats(spec.InitContainers, st.InitContainerStatuses)
|
||||
cReady += iReady
|
||||
allCounts := len(spec.Containers) + iTerminated
|
||||
rgr, rgt := p.readinessGateStats(spec, &st)
|
||||
ready := hasPodReadyCondition(st.Conditions)
|
||||
|
||||
var ccmx []mv1beta1.ContainerMetrics
|
||||
if pwm.MX != nil {
|
||||
|
|
@ -201,7 +203,7 @@ func (p *Pod) defaultRow(pwm *PodWithMetrics, row *model1.Row) error {
|
|||
asReadinessGate(spec, &st),
|
||||
p.mapQOS(st.QOSClass),
|
||||
mapToStr(pwm.Raw.GetLabels()),
|
||||
AsStatus(p.diagnose(phase, cReady, allCounts)),
|
||||
AsStatus(p.diagnose(phase, cReady, allCounts, ready, rgr, rgt)),
|
||||
ToAge(pwm.Raw.GetCreationTimestamp()),
|
||||
}
|
||||
|
||||
|
|
@ -234,16 +236,25 @@ func (p *Pod) Healthy(_ context.Context, o any) error {
|
|||
cr += icr
|
||||
ct += ict
|
||||
|
||||
return p.diagnose(phase, cr, ct)
|
||||
ready := hasPodReadyCondition(st.Conditions)
|
||||
rgr, rgt := p.readinessGateStats(spec, &st)
|
||||
|
||||
return p.diagnose(phase, cr, ct, ready, rgr, rgt)
|
||||
}
|
||||
|
||||
func (*Pod) diagnose(phase string, cr, ct int) error {
|
||||
func (*Pod) diagnose(phase string, cr, ct int, ready bool, rgr, rgt int) error {
|
||||
if phase == Completed {
|
||||
return nil
|
||||
}
|
||||
if cr != ct || ct == 0 {
|
||||
return fmt.Errorf("container ready check failed: %d of %d", cr, ct)
|
||||
}
|
||||
if rgt > 0 && rgr != rgt {
|
||||
return fmt.Errorf("readiness gate check failed: %d of %d", rgr, rgt)
|
||||
}
|
||||
if !ready {
|
||||
return fmt.Errorf("pod condition ready is false")
|
||||
}
|
||||
if phase == Terminating {
|
||||
return fmt.Errorf("pod is terminating")
|
||||
}
|
||||
|
|
@ -428,6 +439,20 @@ func (*Pod) initContainerStats(cc []v1.Container, cos []v1.ContainerStatus) (rea
|
|||
return
|
||||
}
|
||||
|
||||
func (*Pod) readinessGateStats(spec *v1.PodSpec, st *v1.PodStatus) (ready, total int) {
|
||||
total = len(spec.ReadinessGates)
|
||||
for _, readinessGate := range spec.ReadinessGates {
|
||||
for _, condition := range st.Conditions {
|
||||
if condition.Type == readinessGate.ConditionType {
|
||||
if condition.Status == "True" {
|
||||
ready++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Phase reports the given pod phase.
|
||||
func (p *Pod) Phase(dt *metav1.Time, spec *v1.PodSpec, st *v1.PodStatus) string {
|
||||
status := string(st.Phase)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/derailed/k9s/internal/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
res "k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -629,6 +630,149 @@ func Test_podRequests(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func Test_readinessGateStats(t *testing.T) {
|
||||
const (
|
||||
gate1 = "k9s.derailed.com/gate1"
|
||||
gate2 = "k9s.derailed.com/gate2"
|
||||
)
|
||||
|
||||
uu := map[string]struct {
|
||||
spec *v1.PodSpec
|
||||
st *v1.PodStatus
|
||||
r int
|
||||
t int
|
||||
}{
|
||||
"empty": {
|
||||
spec: &v1.PodSpec{},
|
||||
st: &v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}},
|
||||
},
|
||||
r: 0,
|
||||
t: 0,
|
||||
},
|
||||
"single": {
|
||||
spec: &v1.PodSpec{
|
||||
ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}},
|
||||
},
|
||||
st: &v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}},
|
||||
},
|
||||
r: 1,
|
||||
t: 1,
|
||||
},
|
||||
"multiple": {
|
||||
spec: &v1.PodSpec{
|
||||
ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
|
||||
},
|
||||
st: &v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: gate2, Status: v1.ConditionTrue}, {Type: v1.PodReady, Status: v1.ConditionFalse}},
|
||||
},
|
||||
r: 2,
|
||||
t: 2,
|
||||
},
|
||||
"mixed": {
|
||||
spec: &v1.PodSpec{
|
||||
ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
|
||||
},
|
||||
st: &v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: gate2, Status: v1.ConditionFalse}, {Type: v1.PodReady, Status: v1.ConditionTrue}},
|
||||
},
|
||||
r: 1,
|
||||
t: 2,
|
||||
},
|
||||
"missing": {
|
||||
spec: &v1.PodSpec{
|
||||
ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
|
||||
},
|
||||
st: &v1.PodStatus{
|
||||
Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: v1.PodReady, Status: v1.ConditionTrue}},
|
||||
},
|
||||
r: 1,
|
||||
t: 2,
|
||||
},
|
||||
}
|
||||
|
||||
var p Pod
|
||||
for k := range uu {
|
||||
u := uu[k]
|
||||
t.Run(k, func(t *testing.T) {
|
||||
ready, total := p.readinessGateStats(u.spec, u.st)
|
||||
assert.Equal(t, u.r, ready)
|
||||
assert.Equal(t, u.t, total)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_diagnose(t *testing.T) {
|
||||
uu := map[string]struct {
|
||||
phase string
|
||||
cr, ct int
|
||||
ready bool
|
||||
rgr, rgt int
|
||||
err string
|
||||
}{
|
||||
"completed": {
|
||||
phase: Completed,
|
||||
cr: 0,
|
||||
ct: 1,
|
||||
ready: true,
|
||||
rgr: 0,
|
||||
rgt: 0,
|
||||
err: "",
|
||||
},
|
||||
"container-ready-check-failed": {
|
||||
phase: "Running",
|
||||
cr: 1,
|
||||
ct: 2,
|
||||
ready: true,
|
||||
rgr: 1,
|
||||
rgt: 2,
|
||||
err: "container ready check failed: 1 of 2",
|
||||
},
|
||||
"readiness-gate-check-failed": {
|
||||
phase: "Running",
|
||||
cr: 1,
|
||||
ct: 1,
|
||||
ready: true,
|
||||
rgr: 1,
|
||||
rgt: 2,
|
||||
err: "readiness gate check failed: 1 of 2",
|
||||
},
|
||||
"pod-condition-ready-false": {
|
||||
phase: "Running",
|
||||
cr: 1,
|
||||
ct: 1,
|
||||
ready: false,
|
||||
rgr: 0,
|
||||
rgt: 0,
|
||||
err: "pod condition ready is false",
|
||||
},
|
||||
"pod-terminating": {
|
||||
phase: "Terminating",
|
||||
cr: 1,
|
||||
ct: 1,
|
||||
ready: true,
|
||||
rgr: 1,
|
||||
rgt: 1,
|
||||
err: "pod is terminating",
|
||||
},
|
||||
}
|
||||
|
||||
var p Pod
|
||||
for k := range uu {
|
||||
u := uu[k]
|
||||
t.Run(k, func(t *testing.T) {
|
||||
err := p.diagnose(u.phase, u.cr, u.ct, u.ready, u.rgr, u.rgt)
|
||||
if u.err == "" {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), u.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers...
|
||||
|
||||
func makeContainer(n string, restartable bool, rc, rm, lc, lm string) v1.Container {
|
||||
|
|
|
|||
Loading…
Reference in New Issue