Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
dprotaso committed Jan 17, 2024
1 parent 752314e commit a18e776
Show file tree
Hide file tree
Showing 8 changed files with 58 additions and 106 deletions.
50 changes: 31 additions & 19 deletions pkg/apis/serving/k8s_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,28 +51,40 @@ func TransformDeploymentStatus(ds *appsv1.DeploymentStatus) *duckv1.Status {
// below, we'll overwrite this.
depCondSet.Manage(s).MarkTrue(DeploymentConditionReplicaSetReady)

for _, cond := range ds.Conditions {
// TODO(jonjohnsonjr): Should we care about appsv1.DeploymentAvailable here?
switch cond.Type {
case appsv1.DeploymentProgressing:
switch cond.Status {
case corev1.ConditionUnknown:
depCondSet.Manage(s).MarkUnknown(DeploymentConditionProgressing, cond.Reason, cond.Message)
case corev1.ConditionTrue:
depCondSet.Manage(s).MarkTrue(DeploymentConditionProgressing)
case corev1.ConditionFalse:
depCondSet.Manage(s).MarkFalse(DeploymentConditionProgressing, cond.Reason, cond.Message)
conds := []appsv1.DeploymentConditionType{
appsv1.DeploymentProgressing,
appsv1.DeploymentReplicaFailure,
}

for _, wantType := range conds {
for _, cond := range ds.Conditions {
if wantType != cond.Type {
continue
}
case appsv1.DeploymentReplicaFailure:
switch cond.Status {
case corev1.ConditionUnknown:
depCondSet.Manage(s).MarkUnknown(DeploymentConditionReplicaSetReady, cond.Reason, cond.Message)
case corev1.ConditionTrue:
depCondSet.Manage(s).MarkFalse(DeploymentConditionReplicaSetReady, cond.Reason, cond.Message)
case corev1.ConditionFalse:
depCondSet.Manage(s).MarkTrue(DeploymentConditionReplicaSetReady)

switch cond.Type {
case appsv1.DeploymentProgressing:
switch cond.Status {
case corev1.ConditionUnknown:
depCondSet.Manage(s).MarkUnknown(DeploymentConditionProgressing, cond.Reason, cond.Message)
case corev1.ConditionTrue:
depCondSet.Manage(s).MarkTrue(DeploymentConditionProgressing)
case corev1.ConditionFalse:
depCondSet.Manage(s).MarkFalse(DeploymentConditionProgressing, cond.Reason, cond.Message)
}
case appsv1.DeploymentReplicaFailure:
switch cond.Status {
case corev1.ConditionUnknown:
depCondSet.Manage(s).MarkUnknown(DeploymentConditionReplicaSetReady, cond.Reason, cond.Message)
case corev1.ConditionTrue:
depCondSet.Manage(s).MarkFalse(DeploymentConditionReplicaSetReady, cond.Reason, cond.Message)
case corev1.ConditionFalse:
depCondSet.Manage(s).MarkTrue(DeploymentConditionReplicaSetReady)
}
}

}
}

return s
}
7 changes: 0 additions & 7 deletions pkg/apis/serving/v1/revision_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package v1
import (
"time"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
net "knative.dev/networking/pkg/apis/networking"
"knative.dev/pkg/kmeta"
Expand Down Expand Up @@ -144,9 +143,3 @@ func (rs *RevisionStatus) IsActivationRequired() bool {
c := revisionCondSet.Manage(rs).GetCondition(RevisionConditionActive)
return c != nil && c.Status != corev1.ConditionTrue
}

// IsReplicaSetFailure returns true if the deployment replicaset failed to create
func (rs *RevisionStatus) IsReplicaSetFailure(deploymentStatus *appsv1.DeploymentStatus) bool {
ds := serving.TransformDeploymentStatus(deploymentStatus)
return ds != nil && ds.GetCondition(serving.DeploymentConditionReplicaSetReady).IsFalse()
}
43 changes: 0 additions & 43 deletions pkg/apis/serving/v1/revision_helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"testing"
"time"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"

Expand Down Expand Up @@ -268,45 +267,3 @@ func TestSetRoutingState(t *testing.T) {
t.Error("Expected default value for unparsable annotationm but got:", got)
}
}

func TestIsReplicaSetFailure(t *testing.T) {
revisionStatus := RevisionStatus{}
cases := []struct {
name string
status appsv1.DeploymentStatus
IsReplicaSetFailure bool
}{{
name: "empty deployment status should not be a failure",
status: appsv1.DeploymentStatus{},
}, {
name: "Ready deployment status should not be a failure",
status: appsv1.DeploymentStatus{
Conditions: []appsv1.DeploymentCondition{{
Type: appsv1.DeploymentAvailable, Status: corev1.ConditionTrue,
}},
},
}, {
name: "ReplicasetFailure true should be a failure",
status: appsv1.DeploymentStatus{
Conditions: []appsv1.DeploymentCondition{{
Type: appsv1.DeploymentReplicaFailure, Status: corev1.ConditionTrue,
}},
},
IsReplicaSetFailure: true,
}, {
name: "ReplicasetFailure false should not be a failure",
status: appsv1.DeploymentStatus{
Conditions: []appsv1.DeploymentCondition{{
Type: appsv1.DeploymentReplicaFailure, Status: corev1.ConditionFalse,
}},
},
}}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got, want := revisionStatus.IsReplicaSetFailure(&tc.status), tc.IsReplicaSetFailure; got != want {
t.Errorf("IsReplicaSetFailure = %v, want: %v", got, want)
}
})
}
}
8 changes: 0 additions & 8 deletions pkg/apis/serving/v1/revision_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,14 +222,6 @@ func (rs *RevisionStatus) PropagateAutoscalerStatus(ps *autoscalingv1alpha1.PodA
rs.MarkActiveFalse(cond.Reason, cond.Message)
case corev1.ConditionTrue:
rs.MarkActiveTrue()

// Precondition for PA being active is SKS being active and
// that implies that |service.endpoints| > 0.
//
// Note: This is needed for backwards compatibility as we're adding the new
// ScaleTargetInitialized condition to gate readiness.
rs.MarkResourcesAvailableTrue()
rs.MarkContainerHealthyTrue()
}
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/reconciler/revision/cruds.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func (c *Reconciler) createImageCache(ctx context.Context, rev *v1.Revision, con
return c.cachingclient.CachingV1alpha1().Images(image.Namespace).Create(ctx, image, metav1.CreateOptions{})
}

func (c *Reconciler) createPA(ctx context.Context, rev *v1.Revision) (*autoscalingv1alpha1.PodAutoscaler, error) {
pa := resources.MakePA(rev)
func (c *Reconciler) createPA(ctx context.Context, rev *v1.Revision, deployment *appsv1.Deployment) (*autoscalingv1alpha1.PodAutoscaler, error) {
pa := resources.MakePA(rev, deployment)
return c.client.AutoscalingV1alpha1().PodAutoscalers(pa.Namespace).Create(ctx, pa, metav1.CreateOptions{})
}
33 changes: 14 additions & 19 deletions pkg/reconciler/revision/reconcile_resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ func (c *Reconciler) reconcileDeployment(ctx context.Context, rev *v1.Revision)
// Deployment does not exist. Create it.
rev.Status.MarkResourcesAvailableUnknown(v1.ReasonDeploying, "")
rev.Status.MarkContainerHealthyUnknown(v1.ReasonDeploying, "")
deployment, err = c.createDeployment(ctx, rev)
if err != nil {

if _, err = c.createDeployment(ctx, rev); err != nil {
return fmt.Errorf("failed to create deployment %q: %w", deploymentName, err)
}

logger.Infof("Created deployment %q", deploymentName)
return nil
} else if err != nil {
return fmt.Errorf("failed to get deployment %q: %w", deploymentName, err)
} else if !metav1.IsControlledBy(deployment, rev) {
Expand All @@ -66,27 +68,17 @@ func (c *Reconciler) reconcileDeployment(ctx context.Context, rev *v1.Revision)
if err != nil {
return fmt.Errorf("failed to update deployment %q: %w", deploymentName, err)
}

// Now that we have a Deployment, determine whether there is any relevant
// status to surface in the Revision.
//
// TODO(jonjohnsonjr): Should we check Generation != ObservedGeneration?
// The autoscaler mutates the deployment pretty often, which would cause us
// to flip back and forth between Ready and Unknown every time we scale up
// or down.
if !rev.Status.IsActivationRequired() {
rev.Status.PropagateDeploymentStatus(&deployment.Status)
}
}

// If the replicaset is failing we assume its an error we have to surface
if rev.Status.IsReplicaSetFailure(&deployment.Status) {
rev.Status.PropagateDeploymentStatus(&deployment.Status)
if *deployment.Spec.Replicas == 0 {
return nil
}

// want replicas > 0 => (active == true || active == unknown (scaling up for the first time))
rev.Status.PropagateDeploymentStatus(&deployment.Status)

// If a container keeps crashing (no active pods in the deployment although we want some)
if *deployment.Spec.Replicas > 0 && deployment.Status.AvailableReplicas == 0 {
if deployment.Status.AvailableReplicas == 0 {
pods, err := c.kubeclient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(deployment.Spec.Selector)})
if err != nil {
logger.Errorw("Error getting pods", zap.Error(err))
Expand Down Expand Up @@ -151,14 +143,17 @@ func (c *Reconciler) reconcileImageCache(ctx context.Context, rev *v1.Revision)

func (c *Reconciler) reconcilePA(ctx context.Context, rev *v1.Revision) error {
ns := rev.Namespace
deploymentName := resourcenames.Deployment(rev)
deployment, _ := c.deploymentLister.Deployments(ns).Get(deploymentName)

paName := resourcenames.PA(rev)
logger := logging.FromContext(ctx)
logger.Info("Reconciling PA: ", paName)

pa, err := c.podAutoscalerLister.PodAutoscalers(ns).Get(paName)
if apierrs.IsNotFound(err) {
// PA does not exist. Create it.
pa, err = c.createPA(ctx, rev)
pa, err = c.createPA(ctx, rev, deployment)
if err != nil {
return fmt.Errorf("failed to create PA %q: %w", paName, err)
}
Expand All @@ -173,7 +168,7 @@ func (c *Reconciler) reconcilePA(ctx context.Context, rev *v1.Revision) error {

// Perhaps tha PA spec changed underneath ourselves?
// We no longer require immutability, so need to reconcile PA each time.
tmpl := resources.MakePA(rev)
tmpl := resources.MakePA(rev, deployment)
logger.Debugf("Desired PASpec: %#v", tmpl.Spec)
if !equality.Semantic.DeepEqual(tmpl.Spec, pa.Spec) {
diff, _ := kmp.SafeDiff(tmpl.Spec, pa.Spec) // Can't realistically fail on PASpec.
Expand Down
17 changes: 10 additions & 7 deletions pkg/reconciler/revision/resources/pa.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package resources

import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand All @@ -28,7 +29,7 @@ import (
)

// MakePA makes a Knative Pod Autoscaler resource from a revision.
func MakePA(rev *v1.Revision) *autoscalingv1alpha1.PodAutoscaler {
func MakePA(rev *v1.Revision, deployment *appsv1.Deployment) *autoscalingv1alpha1.PodAutoscaler {
return &autoscalingv1alpha1.PodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
Name: names.PA(rev),
Expand All @@ -45,21 +46,23 @@ func MakePA(rev *v1.Revision) *autoscalingv1alpha1.PodAutoscaler {
Name: names.Deployment(rev),
},
ProtocolType: rev.GetProtocol(),
Reachability: reachability(rev),
Reachability: reachability(rev, deployment),
},
}
}

func reachability(rev *v1.Revision) autoscalingv1alpha1.ReachabilityType {
// check infra failures
func reachability(rev *v1.Revision, deployment *appsv1.Deployment) autoscalingv1alpha1.ReachabilityType {
// // check infra failures
conds := []apis.ConditionType{
v1.RevisionConditionResourcesAvailable,
v1.RevisionConditionContainerHealthy,
}

for _, cond := range conds {
if c := rev.Status.GetCondition(cond); c != nil && c.IsFalse() {
return autoscalingv1alpha1.ReachabilityUnreachable
if deployment != nil && deployment.Status.ReadyReplicas == 0 {
for _, cond := range conds {
if c := rev.Status.GetCondition(cond); c != nil && c.IsFalse() {
return autoscalingv1alpha1.ReachabilityUnreachable
}
}
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/reconciler/revision/table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -893,7 +893,7 @@ func imageInit(namespace, name string, co ...configOption) []runtime.Object {

func pa(namespace, name string, ko ...PodAutoscalerOption) *autoscalingv1alpha1.PodAutoscaler {
rev := Revision(namespace, name)
k := resources.MakePA(rev)
k := resources.MakePA(rev, nil)

for _, opt := range ko {
opt(k)
Expand Down

0 comments on commit a18e776

Please sign in to comment.