fix: consider readinessGates + ready condition in diagnose (#3556)

* Consider readinessGates + ready condition in diagnose

Now that readiness gates are supported, those should be considered when displaying a pod healthiness

Additionally consider the pod ready condition status. It should match the && of the containers' ready condition and readiness gates but I've actually observed that in some cases, it can be false with the containers reporting ready true.

* Update pod.go

* Update pod.go

fix lint

* Update pod.go

lint

* add tests for diagnose and readinessGateStats

* lint error

internal/render/pod.go

@@ -165,6 +165,8 @@ func (p *Pod) defaultRow(pwm *PodWithMetrics, row *model1.Row) error {
 	iReady, iTerminated, iRestarts := p.initContainerStats(spec.InitContainers, st.InitContainerStatuses)
 	cReady += iReady
 	allCounts := len(spec.Containers) + iTerminated
+	rgr, rgt := p.readinessGateStats(spec, &st)
+	ready := hasPodReadyCondition(st.Conditions)
 
 	var ccmx []mv1beta1.ContainerMetrics
 	if pwm.MX != nil {
@@ -201,7 +203,7 @@ func (p *Pod) defaultRow(pwm *PodWithMetrics, row *model1.Row) error {
 		asReadinessGate(spec, &st),
 		p.mapQOS(st.QOSClass),
 		mapToStr(pwm.Raw.GetLabels()),
-		AsStatus(p.diagnose(phase, cReady, allCounts)),
+		AsStatus(p.diagnose(phase, cReady, allCounts, ready, rgr, rgt)),
 		ToAge(pwm.Raw.GetCreationTimestamp()),
 	}
 
@@ -234,16 +236,25 @@ func (p *Pod) Healthy(_ context.Context, o any) error {
 	cr += icr
 	ct += ict
 
-	return p.diagnose(phase, cr, ct)
+	ready := hasPodReadyCondition(st.Conditions)
+	rgr, rgt := p.readinessGateStats(spec, &st)
+
+	return p.diagnose(phase, cr, ct, ready, rgr, rgt)
 }
 
-func (*Pod) diagnose(phase string, cr, ct int) error {
+func (*Pod) diagnose(phase string, cr, ct int, ready bool, rgr, rgt int) error {
 	if phase == Completed {
 		return nil
 	}
 	if cr != ct || ct == 0 {
 		return fmt.Errorf("container ready check failed: %d of %d", cr, ct)
 	}
+	if rgt > 0 && rgr != rgt {
+		return fmt.Errorf("readiness gate check failed: %d of %d", rgr, rgt)
+	}
+	if !ready {
+		return fmt.Errorf("pod condition ready is false")
+	}
 	if phase == Terminating {
 		return fmt.Errorf("pod is terminating")
 	}
@@ -428,6 +439,20 @@ func (*Pod) initContainerStats(cc []v1.Container, cos []v1.ContainerStatus) (rea
 	return
 }
 
+func (*Pod) readinessGateStats(spec *v1.PodSpec, st *v1.PodStatus) (ready, total int) {
+	total = len(spec.ReadinessGates)
+	for _, readinessGate := range spec.ReadinessGates {
+		for _, condition := range st.Conditions {
+			if condition.Type == readinessGate.ConditionType {
+				if condition.Status == "True" {
+					ready++
+				}
+			}
+		}
+	}
+	return
+}
+
 // Phase reports the given pod phase.
 func (p *Pod) Phase(dt *metav1.Time, spec *v1.PodSpec, st *v1.PodStatus) string {
 	status := string(st.Phase)

internal/render/pod_int_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/derailed/k9s/internal/client"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	res "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -629,6 +630,149 @@ func Test_podRequests(t *testing.T) {
 	}
 }
 
+func Test_readinessGateStats(t *testing.T) {
+	const (
+		gate1 = "k9s.derailed.com/gate1"
+		gate2 = "k9s.derailed.com/gate2"
+	)
+
+	uu := map[string]struct {
+		spec *v1.PodSpec
+		st   *v1.PodStatus
+		r    int
+		t    int
+	}{
+		"empty": {
+			spec: &v1.PodSpec{},
+			st: &v1.PodStatus{
+				Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}},
+			},
+			r: 0,
+			t: 0,
+		},
+		"single": {
+			spec: &v1.PodSpec{
+				ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}},
+			},
+			st: &v1.PodStatus{
+				Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}},
+			},
+			r: 1,
+			t: 1,
+		},
+		"multiple": {
+			spec: &v1.PodSpec{
+				ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
+			},
+			st: &v1.PodStatus{
+				Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: gate2, Status: v1.ConditionTrue}, {Type: v1.PodReady, Status: v1.ConditionFalse}},
+			},
+			r: 2,
+			t: 2,
+		},
+		"mixed": {
+			spec: &v1.PodSpec{
+				ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
+			},
+			st: &v1.PodStatus{
+				Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: gate2, Status: v1.ConditionFalse}, {Type: v1.PodReady, Status: v1.ConditionTrue}},
+			},
+			r: 1,
+			t: 2,
+		},
+		"missing": {
+			spec: &v1.PodSpec{
+				ReadinessGates: []v1.PodReadinessGate{{ConditionType: gate1}, {ConditionType: gate2}},
+			},
+			st: &v1.PodStatus{
+				Conditions: []v1.PodCondition{{Type: gate1, Status: v1.ConditionTrue}, {Type: v1.PodReady, Status: v1.ConditionTrue}},
+			},
+			r: 1,
+			t: 2,
+		},
+	}
+
+	var p Pod
+	for k := range uu {
+		u := uu[k]
+		t.Run(k, func(t *testing.T) {
+			ready, total := p.readinessGateStats(u.spec, u.st)
+			assert.Equal(t, u.r, ready)
+			assert.Equal(t, u.t, total)
+		})
+	}
+}
+
+func Test_diagnose(t *testing.T) {
+	uu := map[string]struct {
+		phase    string
+		cr, ct   int
+		ready    bool
+		rgr, rgt int
+		err      string
+	}{
+		"completed": {
+			phase: Completed,
+			cr:    0,
+			ct:    1,
+			ready: true,
+			rgr:   0,
+			rgt:   0,
+			err:   "",
+		},
+		"container-ready-check-failed": {
+			phase: "Running",
+			cr:    1,
+			ct:    2,
+			ready: true,
+			rgr:   1,
+			rgt:   2,
+			err:   "container ready check failed: 1 of 2",
+		},
+		"readiness-gate-check-failed": {
+			phase: "Running",
+			cr:    1,
+			ct:    1,
+			ready: true,
+			rgr:   1,
+			rgt:   2,
+			err:   "readiness gate check failed: 1 of 2",
+		},
+		"pod-condition-ready-false": {
+			phase: "Running",
+			cr:    1,
+			ct:    1,
+			ready: false,
+			rgr:   0,
+			rgt:   0,
+			err:   "pod condition ready is false",
+		},
+		"pod-terminating": {
+			phase: "Terminating",
+			cr:    1,
+			ct:    1,
+			ready: true,
+			rgr:   1,
+			rgt:   1,
+			err:   "pod is terminating",
+		},
+	}
+
+	var p Pod
+	for k := range uu {
+		u := uu[k]
+		t.Run(k, func(t *testing.T) {
+			err := p.diagnose(u.phase, u.cr, u.ct, u.ready, u.rgr, u.rgt)
+			if u.err == "" {
+				assert.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), u.err)
+			}
+		})
+	}
+}
+
 // Helpers...
 
 func makeContainer(n string, restartable bool, rc, rm, lc, lm string) v1.Container {