working commit

This commit is contained in:
2026-03-13 19:02:42 +02:00
parent bebbf79c7a
commit 5c1da77f4c
1329 changed files with 314708 additions and 39 deletions
File diff suppressed because it is too large Load Diff
+69
View File
@@ -0,0 +1,69 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import (
"sync"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes/scheme"
)
var k8sNativeScheme *runtime.Scheme
var k8sNativeSchemeOnce sync.Once
// AsVersioned converts the given info into a runtime.Object with the correct
// group and version set
func AsVersioned(info *resource.Info) runtime.Object {
return convertWithMapper(info.Object, info.Mapping)
}
// convertWithMapper converts the given object with the optional provided
// RESTMapping. If no mapping is provided, the default schema versioner is used
func convertWithMapper(obj runtime.Object, mapping *meta.RESTMapping) runtime.Object {
s := kubernetesNativeScheme()
var gv = runtime.GroupVersioner(schema.GroupVersions(s.PrioritizedVersionsAllGroups()))
if mapping != nil {
gv = mapping.GroupVersionKind.GroupVersion()
}
if obj, err := runtime.ObjectConvertor(s).ConvertToVersion(obj, gv); err == nil {
return obj
}
return obj
}
// kubernetesNativeScheme returns a clean *runtime.Scheme with _only_ Kubernetes
// native resources added to it. This is required to break free of custom resources
// that may have been added to scheme.Scheme due to Helm being used as a package in
// combination with e.g. a versioned kube client. If we would not do this, the client
// may attempt to perform e.g. a 3-way-merge strategy patch for custom resources.
func kubernetesNativeScheme() *runtime.Scheme {
k8sNativeSchemeOnce.Do(func() {
k8sNativeScheme = runtime.NewScheme()
scheme.AddToScheme(k8sNativeScheme)
// API extensions are not in the above scheme set,
// and must thus be added separately.
apiextensionsv1beta1.AddToScheme(k8sNativeScheme)
apiextensionsv1.AddToScheme(k8sNativeScheme)
})
return k8sNativeScheme
}
+55
View File
@@ -0,0 +1,55 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import (
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubectl/pkg/validation"
)
// Factory provides abstractions that allow the Kubectl command to be extended across multiple types
// of resources and different API sets.
// This interface is a minimal copy of the kubectl Factory interface containing only the functions
// needed by Helm. Since Kubernetes Go APIs, including interfaces, can change in any minor release
// this interface is not covered by the Helm backwards compatibility guarantee. The reasons for the
// minimal copy is that it does not include the full interface. Changes or additions to functions
// Helm does not need are not impacted or exposed. This minimizes the impact of Kubernetes changes
// being exposed.
type Factory interface {
// ToRESTConfig returns restconfig
ToRESTConfig() (*rest.Config, error)
// ToRawKubeConfigLoader return kubeconfig loader as-is
ToRawKubeConfigLoader() clientcmd.ClientConfig
// DynamicClient returns a dynamic client ready for use
DynamicClient() (dynamic.Interface, error)
// KubernetesClientSet gives you back an external clientset
KubernetesClientSet() (*kubernetes.Clientset, error)
// NewBuilder returns an object that assists in loading objects from both disk and the server
// and which implements the common patterns for CLI interactions with generic resources.
NewBuilder() *resource.Builder
// Returns a schema that can validate objects stored on disk.
Validator(validationDirective string) (validation.Schema, error)
}
+112
View File
@@ -0,0 +1,112 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"io"
"time"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// Interface represents a client capable of communicating with the Kubernetes API.
//
// A KubernetesClient must be concurrency safe.
type Interface interface {
// Get details of deployed resources.
// The first argument is a list of resources to get. The second argument
// specifies if related pods should be fetched. For example, the pods being
// managed by a deployment.
Get(resources ResourceList, related bool) (map[string][]runtime.Object, error)
// Create creates one or more resources.
Create(resources ResourceList, options ...ClientCreateOption) (*Result, error)
// Delete destroys one or more resources using the specified deletion propagation policy.
// The 'policy' parameter determines how child resources are handled during deletion.
Delete(resources ResourceList, policy metav1.DeletionPropagation) (*Result, []error)
// Update updates one or more resources or creates the resource
// if it doesn't exist.
Update(original, target ResourceList, options ...ClientUpdateOption) (*Result, error)
// Build creates a resource list from a Reader.
//
// Reader must contain a YAML stream (one or more YAML documents separated
// by "\n---\n")
//
// Validates against OpenAPI schema if validate is true.
Build(reader io.Reader, validate bool) (ResourceList, error)
// IsReachable checks whether the client is able to connect to the cluster.
IsReachable() error
// GetWaiter gets the Kube.Waiter.
GetWaiter(ws WaitStrategy) (Waiter, error)
// GetPodList lists all pods that match the specified listOptions
GetPodList(namespace string, listOptions metav1.ListOptions) (*v1.PodList, error)
// OutputContainerLogsForPodList outputs the logs for a pod list
OutputContainerLogsForPodList(podList *v1.PodList, namespace string, writerFunc func(namespace, pod, container string) io.Writer) error
// BuildTable creates a resource list from a Reader. This differs from
// Interface.Build() in that a table kind is returned. A table is useful
// if you want to use a printer to display the information.
//
// Reader must contain a YAML stream (one or more YAML documents separated
// by "\n---\n")
//
// Validates against OpenAPI schema if validate is true.
// TODO Helm 4: Integrate into Build with an argument
BuildTable(reader io.Reader, validate bool) (ResourceList, error)
}
// Waiter defines methods related to waiting for resource states.
type Waiter interface {
// Wait waits up to the given timeout for the specified resources to be ready.
Wait(resources ResourceList, timeout time.Duration) error
// WaitWithJobs wait up to the given timeout for the specified resources to be ready, including jobs.
WaitWithJobs(resources ResourceList, timeout time.Duration) error
// WaitForDelete wait up to the given timeout for the specified resources to be deleted.
WaitForDelete(resources ResourceList, timeout time.Duration) error
// WatchUntilReady watches the resources given and waits until it is ready.
//
// This method is mainly for hook implementations. It watches for a resource to
// hit a particular milestone. The milestone depends on the Kind.
//
// For Jobs, "ready" means the Job ran to completion (exited without error).
// For Pods, "ready" means the Pod phase is marked "succeeded".
// For all other kinds, it means the kind was created or modified without
// error.
WatchUntilReady(resources ResourceList, timeout time.Duration) error
}
// InterfaceWaitOptions defines an interface that extends Interface with
// methods that accept wait options.
//
// TODO Helm 5: Remove InterfaceWaitOptions and integrate its method(s) into the Interface.
type InterfaceWaitOptions interface {
// GetWaiter gets the Kube.Waiter with options.
GetWaiterWithOptions(ws WaitStrategy, opts ...WaitOption) (Waiter, error)
}
var _ InterfaceWaitOptions = (*Client)(nil)
+82
View File
@@ -0,0 +1,82 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"context"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/engine"
)
// WaitOption is a function that configures an option for waiting on resources.
type WaitOption func(*waitOptions)
// WithWaitContext sets the context for waiting on resources.
// If unset, context.Background() will be used.
func WithWaitContext(ctx context.Context) WaitOption {
return func(wo *waitOptions) {
wo.ctx = ctx
}
}
// WithWatchUntilReadyMethodContext sets the context specifically for the WatchUntilReady method.
// If unset, the context set by `WithWaitContext` will be used (falling back to `context.Background()`).
func WithWatchUntilReadyMethodContext(ctx context.Context) WaitOption {
return func(wo *waitOptions) {
wo.watchUntilReadyCtx = ctx
}
}
// WithWaitMethodContext sets the context specifically for the Wait method.
// If unset, the context set by `WithWaitContext` will be used (falling back to `context.Background()`).
func WithWaitMethodContext(ctx context.Context) WaitOption {
return func(wo *waitOptions) {
wo.waitCtx = ctx
}
}
// WithWaitWithJobsMethodContext sets the context specifically for the WaitWithJobs method.
// If unset, the context set by `WithWaitContext` will be used (falling back to `context.Background()`).
func WithWaitWithJobsMethodContext(ctx context.Context) WaitOption {
return func(wo *waitOptions) {
wo.waitWithJobsCtx = ctx
}
}
// WithWaitForDeleteMethodContext sets the context specifically for the WaitForDelete method.
// If unset, the context set by `WithWaitContext` will be used (falling back to `context.Background()`).
func WithWaitForDeleteMethodContext(ctx context.Context) WaitOption {
return func(wo *waitOptions) {
wo.waitForDeleteCtx = ctx
}
}
// WithKStatusReaders sets the status readers to be used while waiting on resources.
func WithKStatusReaders(readers ...engine.StatusReader) WaitOption {
return func(wo *waitOptions) {
wo.statusReaders = readers
}
}
type waitOptions struct {
ctx context.Context
watchUntilReadyCtx context.Context
waitCtx context.Context
waitWithJobsCtx context.Context
waitForDeleteCtx context.Context
statusReaders []engine.StatusReader
}
+466
View File
@@ -0,0 +1,466 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import (
"context"
"fmt"
"log/slog"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
deploymentutil "helm.sh/helm/v4/internal/third_party/k8s.io/kubernetes/deployment/util"
)
// ReadyCheckerOption is a function that configures a ReadyChecker.
type ReadyCheckerOption func(*ReadyChecker)
// PausedAsReady returns a ReadyCheckerOption that configures a ReadyChecker
// to consider paused resources to be ready. For example a Deployment
// with spec.paused equal to true would be considered ready.
func PausedAsReady(pausedAsReady bool) ReadyCheckerOption {
return func(c *ReadyChecker) {
c.pausedAsReady = pausedAsReady
}
}
// CheckJobs returns a ReadyCheckerOption that configures a ReadyChecker
// to consider readiness of Job resources.
func CheckJobs(checkJobs bool) ReadyCheckerOption {
return func(c *ReadyChecker) {
c.checkJobs = checkJobs
}
}
// NewReadyChecker creates a new checker. Passed ReadyCheckerOptions can
// be used to override defaults.
func NewReadyChecker(cl kubernetes.Interface, opts ...ReadyCheckerOption) ReadyChecker {
c := ReadyChecker{
client: cl,
}
for _, opt := range opts {
opt(&c)
}
return c
}
// ReadyChecker is a type that can check core Kubernetes types for readiness.
type ReadyChecker struct {
client kubernetes.Interface
checkJobs bool
pausedAsReady bool
}
// IsReady checks if v is ready. It supports checking readiness for pods,
// deployments, persistent volume claims, services, daemon sets, custom
// resource definitions, stateful sets, replication controllers, jobs (optional),
// and replica sets. All other resource kinds are always considered ready.
//
// IsReady will fetch the latest state of the object from the server prior to
// performing readiness checks, and it will return any error encountered.
func (c *ReadyChecker) IsReady(ctx context.Context, v *resource.Info) (bool, error) {
switch value := AsVersioned(v).(type) {
case *corev1.Pod:
pod, err := c.client.CoreV1().Pods(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil || !c.isPodReady(pod) {
return false, err
}
case *batchv1.Job:
if c.checkJobs {
job, err := c.client.BatchV1().Jobs(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
ready, err := c.jobReady(job)
return ready, err
}
case *appsv1.Deployment:
currentDeployment, err := c.client.AppsV1().Deployments(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
// If paused deployment will never be ready
if currentDeployment.Spec.Paused {
return c.pausedAsReady, nil
}
// Find RS associated with deployment
newReplicaSet, err := deploymentutil.GetNewReplicaSet(currentDeployment, c.client.AppsV1())
if err != nil || newReplicaSet == nil {
return false, err
}
if !c.deploymentReady(newReplicaSet, currentDeployment) {
return false, nil
}
case *corev1.PersistentVolumeClaim:
claim, err := c.client.CoreV1().PersistentVolumeClaims(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.volumeReady(claim) {
return false, nil
}
case *corev1.Service:
svc, err := c.client.CoreV1().Services(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.serviceReady(svc) {
return false, nil
}
case *appsv1.DaemonSet:
ds, err := c.client.AppsV1().DaemonSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.daemonSetReady(ds) {
return false, nil
}
case *apiextv1beta1.CustomResourceDefinition:
if err := v.Get(); err != nil {
return false, err
}
crd := &apiextv1beta1.CustomResourceDefinition{}
if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
return false, err
}
if !c.crdBetaReady(*crd) {
return false, nil
}
case *apiextv1.CustomResourceDefinition:
if err := v.Get(); err != nil {
return false, err
}
crd := &apiextv1.CustomResourceDefinition{}
if err := scheme.Scheme.Convert(v.Object, crd, nil); err != nil {
return false, err
}
if !c.crdReady(*crd) {
return false, nil
}
case *appsv1.StatefulSet:
sts, err := c.client.AppsV1().StatefulSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.statefulSetReady(sts) {
return false, nil
}
case *corev1.ReplicationController:
rc, err := c.client.CoreV1().ReplicationControllers(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.replicationControllerReady(rc) {
return false, nil
}
ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
if !ready || err != nil {
return false, err
}
case *appsv1.ReplicaSet:
rs, err := c.client.AppsV1().ReplicaSets(v.Namespace).Get(ctx, v.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
if !c.replicaSetReady(rs) {
return false, nil
}
ready, err := c.podsReadyForObject(ctx, v.Namespace, value)
if !ready || err != nil {
return false, err
}
}
return true, nil
}
func (c *ReadyChecker) podsReadyForObject(ctx context.Context, namespace string, obj runtime.Object) (bool, error) {
pods, err := c.podsforObject(ctx, namespace, obj)
if err != nil {
return false, err
}
for _, pod := range pods {
if !c.isPodReady(&pod) {
return false, nil
}
}
return true, nil
}
func (c *ReadyChecker) podsforObject(ctx context.Context, namespace string, obj runtime.Object) ([]corev1.Pod, error) {
selector, err := SelectorsForObject(obj)
if err != nil {
return nil, err
}
list, err := getPods(ctx, c.client, namespace, selector.String())
return list, err
}
// isPodReady returns true if a pod is ready; false otherwise.
func (c *ReadyChecker) isPodReady(pod *corev1.Pod) bool {
for _, c := range pod.Status.Conditions {
if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue {
return true
}
}
slog.Debug("Pod is not ready", "namespace", pod.GetNamespace(), "name", pod.GetName())
return false
}
func (c *ReadyChecker) jobReady(job *batchv1.Job) (bool, error) {
if job.Status.Failed > *job.Spec.BackoffLimit {
slog.Debug("Job is failed", "namespace", job.GetNamespace(), "name", job.GetName())
// If a job is failed, it can't recover, so throw an error
return false, fmt.Errorf("job is failed: %s/%s", job.GetNamespace(), job.GetName())
}
if job.Spec.Completions != nil && job.Status.Succeeded < *job.Spec.Completions {
slog.Debug("Job is not completed", "namespace", job.GetNamespace(), "name", job.GetName())
return false, nil
}
slog.Debug("Job is completed", "namespace", job.GetNamespace(), "name", job.GetName())
return true, nil
}
func (c *ReadyChecker) serviceReady(s *corev1.Service) bool {
// ExternalName Services are external to cluster so helm shouldn't be checking to see if they're 'ready' (i.e. have an IP Set)
if s.Spec.Type == corev1.ServiceTypeExternalName {
return true
}
// Ensure that the service cluster IP is not empty
if s.Spec.ClusterIP == "" {
slog.Debug("Service does not have cluster IP address", "namespace", s.GetNamespace(), "name", s.GetName())
return false
}
// This checks if the service has a LoadBalancer and that balancer has an Ingress defined
if s.Spec.Type == corev1.ServiceTypeLoadBalancer {
// do not wait when at least 1 external IP is set
if len(s.Spec.ExternalIPs) > 0 {
slog.Debug("Service has external IP addresses", "namespace", s.GetNamespace(), "name", s.GetName(), "externalIPs", s.Spec.ExternalIPs)
return true
}
if s.Status.LoadBalancer.Ingress == nil {
slog.Debug("Service does not have load balancer ingress IP address", "namespace", s.GetNamespace(), "name", s.GetName())
return false
}
}
slog.Debug("Service is ready", "namespace", s.GetNamespace(), "name", s.GetName(), "clusterIP", s.Spec.ClusterIP, "externalIPs", s.Spec.ExternalIPs)
return true
}
func (c *ReadyChecker) volumeReady(v *corev1.PersistentVolumeClaim) bool {
if v.Status.Phase != corev1.ClaimBound {
slog.Debug("PersistentVolumeClaim is not bound", "namespace", v.GetNamespace(), "name", v.GetName())
return false
}
slog.Debug("PersistentVolumeClaim is bound", "namespace", v.GetNamespace(), "name", v.GetName(), "phase", v.Status.Phase)
return true
}
func (c *ReadyChecker) deploymentReady(rs *appsv1.ReplicaSet, dep *appsv1.Deployment) bool {
// Verify the replicaset readiness
if !c.replicaSetReady(rs) {
return false
}
// Verify the generation observed by the deployment controller matches the spec generation
if dep.Status.ObservedGeneration != dep.Generation {
slog.Debug("Deployment is not ready, observedGeneration does not match spec generation", "namespace", dep.GetNamespace(), "name", dep.GetName(), "actualGeneration", dep.Status.ObservedGeneration, "expectedGeneration", dep.Generation)
return false
}
expectedReady := *dep.Spec.Replicas - deploymentutil.MaxUnavailable(*dep)
if rs.Status.ReadyReplicas < expectedReady {
slog.Debug("Deployment does not have enough pods ready", "namespace", dep.GetNamespace(), "name", dep.GetName(), "readyPods", rs.Status.ReadyReplicas, "totalPods", expectedReady)
return false
}
slog.Debug("Deployment is ready", "namespace", dep.GetNamespace(), "name", dep.GetName(), "readyPods", rs.Status.ReadyReplicas, "totalPods", expectedReady)
return true
}
func (c *ReadyChecker) daemonSetReady(ds *appsv1.DaemonSet) bool {
// Verify the generation observed by the daemonSet controller matches the spec generation
if ds.Status.ObservedGeneration != ds.Generation {
slog.Debug("DaemonSet is not ready, observedGeneration does not match spec generation", "namespace", ds.GetNamespace(), "name", ds.GetName(), "observedGeneration", ds.Status.ObservedGeneration, "expectedGeneration", ds.Generation)
return false
}
// If the update strategy is not a rolling update, there will be nothing to wait for
if ds.Spec.UpdateStrategy.Type != appsv1.RollingUpdateDaemonSetStrategyType {
return true
}
// Make sure all the updated pods have been scheduled
if ds.Status.UpdatedNumberScheduled != ds.Status.DesiredNumberScheduled {
slog.Debug("DaemonSet does not have enough Pods scheduled", "namespace", ds.GetNamespace(), "name", ds.GetName(), "scheduledPods", ds.Status.UpdatedNumberScheduled, "totalPods", ds.Status.DesiredNumberScheduled)
return false
}
maxUnavailable, err := intstr.GetScaledValueFromIntOrPercent(ds.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, int(ds.Status.DesiredNumberScheduled), true)
if err != nil {
// If for some reason the value is invalid, set max unavailable to the
// number of desired replicas. This is the same behavior as the
// `MaxUnavailable` function in deploymentutil
maxUnavailable = int(ds.Status.DesiredNumberScheduled)
}
expectedReady := int(ds.Status.DesiredNumberScheduled) - maxUnavailable
if int(ds.Status.NumberReady) < expectedReady {
slog.Debug("DaemonSet does not have enough Pods ready", "namespace", ds.GetNamespace(), "name", ds.GetName(), "readyPods", ds.Status.NumberReady, "totalPods", expectedReady)
return false
}
slog.Debug("DaemonSet is ready", "namespace", ds.GetNamespace(), "name", ds.GetName(), "readyPods", ds.Status.NumberReady, "totalPods", expectedReady)
return true
}
// Because the v1 extensions API is not available on all supported k8s versions
// yet and because Go doesn't support generics, we need to have a duplicate
// function to support the v1beta1 types
func (c *ReadyChecker) crdBetaReady(crd apiextv1beta1.CustomResourceDefinition) bool {
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextv1beta1.Established:
if cond.Status == apiextv1beta1.ConditionTrue {
return true
}
case apiextv1beta1.NamesAccepted:
if cond.Status == apiextv1beta1.ConditionFalse {
// This indicates a naming conflict, but it's probably not the
// job of this function to fail because of that. Instead,
// we treat it as a success, since the process should be able to
// continue.
return true
}
default:
// intentionally left empty
}
}
return false
}
func (c *ReadyChecker) crdReady(crd apiextv1.CustomResourceDefinition) bool {
for _, cond := range crd.Status.Conditions {
switch cond.Type {
case apiextv1.Established:
if cond.Status == apiextv1.ConditionTrue {
return true
}
case apiextv1.NamesAccepted:
if cond.Status == apiextv1.ConditionFalse {
// This indicates a naming conflict, but it's probably not the
// job of this function to fail because of that. Instead,
// we treat it as a success, since the process should be able to
// continue.
return true
}
default:
// intentionally left empty
}
}
return false
}
func (c *ReadyChecker) statefulSetReady(sts *appsv1.StatefulSet) bool {
// Verify the generation observed by the statefulSet controller matches the spec generation
if sts.Status.ObservedGeneration != sts.Generation {
slog.Debug("StatefulSet is not ready, observedGeneration doest not match spec generation", "namespace", sts.GetNamespace(), "name", sts.GetName(), "actualGeneration", sts.Status.ObservedGeneration, "expectedGeneration", sts.Generation)
return false
}
// If the update strategy is not a rolling update, there will be nothing to wait for
if sts.Spec.UpdateStrategy.Type != appsv1.RollingUpdateStatefulSetStrategyType {
slog.Debug("StatefulSet skipped ready check", "namespace", sts.GetNamespace(), "name", sts.GetName(), "updateStrategy", sts.Spec.UpdateStrategy.Type)
return true
}
// Dereference all the pointers because StatefulSets like them
var partition int
// 1 is the default for replicas if not set
replicas := 1
// For some reason, even if the update strategy is a rolling update, the
// actual rollingUpdate field can be nil. If it is, we can safely assume
// there is no partition value
if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition)
}
if sts.Spec.Replicas != nil {
replicas = int(*sts.Spec.Replicas)
}
// Because an update strategy can use partitioning, we need to calculate the
// number of updated replicas we should have. For example, if the replicas
// is set to 3 and the partition is 2, we'd expect only one pod to be
// updated
expectedReplicas := replicas - partition
// Make sure all the updated pods have been scheduled
if int(sts.Status.UpdatedReplicas) < expectedReplicas {
slog.Debug("StatefulSet does not have enough Pods scheduled", "namespace", sts.GetNamespace(), "name", sts.GetName(), "readyPods", sts.Status.UpdatedReplicas, "totalPods", expectedReplicas)
return false
}
if int(sts.Status.ReadyReplicas) != replicas {
slog.Debug("StatefulSet does not have enough Pods ready", "namespace", sts.GetNamespace(), "name", sts.GetName(), "readyPods", sts.Status.ReadyReplicas, "totalPods", replicas)
return false
}
// This check only makes sense when all partitions are being upgraded otherwise during a
// partitioned rolling upgrade, this condition will never evaluate to true, leading to
// error.
if partition == 0 && sts.Status.CurrentRevision != sts.Status.UpdateRevision {
slog.Debug("StatefulSet is not ready, currentRevision does not match updateRevision", "namespace", sts.GetNamespace(), "name", sts.GetName(), "currentRevision", sts.Status.CurrentRevision, "updateRevision", sts.Status.UpdateRevision)
return false
}
slog.Debug("StatefulSet is ready", "namespace", sts.GetNamespace(), "name", sts.GetName(), "readyPods", sts.Status.ReadyReplicas, "totalPods", replicas)
return true
}
func (c *ReadyChecker) replicationControllerReady(rc *corev1.ReplicationController) bool {
// Verify the generation observed by the replicationController controller matches the spec generation
if rc.Status.ObservedGeneration != rc.Generation {
slog.Debug("ReplicationController is not ready, observedGeneration doest not match spec generation", "namespace", rc.GetNamespace(), "name", rc.GetName(), "actualGeneration", rc.Status.ObservedGeneration, "expectedGeneration", rc.Generation)
return false
}
return true
}
func (c *ReadyChecker) replicaSetReady(rs *appsv1.ReplicaSet) bool {
// Verify the generation observed by the replicaSet controller matches the spec generation
if rs.Status.ObservedGeneration != rs.Generation {
slog.Debug("ReplicaSet is not ready, observedGeneration doest not match spec generation", "namespace", rs.GetNamespace(), "name", rs.GetName(), "actualGeneration", rs.Status.ObservedGeneration, "expectedGeneration", rs.Generation)
return false
}
return true
}
func getPods(ctx context.Context, client kubernetes.Interface, namespace, selector string) ([]corev1.Pod, error) {
list, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
LabelSelector: selector,
})
if err != nil {
return nil, fmt.Errorf("failed to list pods: %w", err)
}
return list.Items, nil
}
+92
View File
@@ -0,0 +1,92 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import "k8s.io/cli-runtime/pkg/resource"
// ResourceList provides convenience methods for comparing collections of Infos.
type ResourceList []*resource.Info
// Append adds an Info to the Result.
func (r *ResourceList) Append(val *resource.Info) {
*r = append(*r, val)
}
// Visit implements resource.Visitor. The visitor stops if fn returns an error.
func (r ResourceList) Visit(fn resource.VisitorFunc) error {
for _, i := range r {
if err := fn(i, nil); err != nil {
return err
}
}
return nil
}
// Filter returns a new Result with Infos that satisfy the predicate fn.
func (r ResourceList) Filter(fn func(*resource.Info) bool) ResourceList {
var result ResourceList
for _, i := range r {
if fn(i) {
result.Append(i)
}
}
return result
}
// Get returns the Info from the result that matches the name and kind.
func (r ResourceList) Get(info *resource.Info) *resource.Info {
for _, i := range r {
if isMatchingInfo(i, info) {
return i
}
}
return nil
}
// Contains checks to see if an object exists.
func (r ResourceList) Contains(info *resource.Info) bool {
for _, i := range r {
if isMatchingInfo(i, info) {
return true
}
}
return false
}
// Difference will return a new Result with objects not contained in rs.
func (r ResourceList) Difference(rs ResourceList) ResourceList {
return r.Filter(func(info *resource.Info) bool {
return !rs.Contains(info)
})
}
// Intersect will return a new Result with objects contained in both Results.
func (r ResourceList) Intersect(rs ResourceList) ResourceList {
return r.Filter(rs.Contains)
}
// isMatchingInfo returns true if infos match on Name, Namespace, Group and Kind.
//
// IMPORTANT: Version is intentionally excluded from the comparison. Resources
// served by the same CRD at different API versions (e.g. v2beta1 vs v2beta2)
// share the same underlying storage in the Kubernetes API server. Comparing
// the full GroupVersionKind causes Difference() to treat a version change as
// a resource removal + addition, which makes Helm delete the resource it just
// created during upgrades. See https://github.com/helm/helm/issues/31768
func isMatchingInfo(a, b *resource.Info) bool {
return a.Name == b.Name && a.Namespace == b.Namespace && a.Mapping.GroupVersionKind.GroupKind() == b.Mapping.GroupVersionKind.GroupKind()
}
+27
View File
@@ -0,0 +1,27 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
// ResourcePolicyAnno is the annotation name for a resource policy
const ResourcePolicyAnno = "helm.sh/resource-policy"
// KeepPolicy is the resource policy type for keep
//
// This resource policy type allows resources to skip being deleted
//
// during an uninstallRelease action.
const KeepPolicy = "keep"
+28
View File
@@ -0,0 +1,28 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
// Result contains the information of created, updated, and deleted resources
// for various kube API calls along with helper methods for using those
// resources
type Result struct {
Created ResourceList
Updated ResourceList
Deleted ResourceList
}
// If needed, we can add methods to the Result type for things like diffing
+80
View File
@@ -0,0 +1,80 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"bytes"
"encoding/json"
"io"
"net/http"
"strings"
)
type RetryingRoundTripper struct {
Wrapped http.RoundTripper
}
func (rt *RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
return rt.roundTrip(req, 1, nil)
}
func (rt *RetryingRoundTripper) roundTrip(req *http.Request, retry int, prevResp *http.Response) (*http.Response, error) {
if retry < 0 {
return prevResp, nil
}
resp, rtErr := rt.Wrapped.RoundTrip(req)
if rtErr != nil {
return resp, rtErr
}
if resp.StatusCode < 500 {
return resp, rtErr
}
if resp.Header.Get("content-type") != "application/json" {
return resp, rtErr
}
b, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return resp, err
}
var ke kubernetesError
r := bytes.NewReader(b)
err = json.NewDecoder(r).Decode(&ke)
r.Seek(0, io.SeekStart)
resp.Body = io.NopCloser(r)
if err != nil {
return resp, err
}
if ke.Code < 500 {
return resp, nil
}
// Matches messages like "etcdserver: leader changed"
if strings.HasSuffix(ke.Message, "etcdserver: leader changed") {
return rt.roundTrip(req, retry-1, resp)
}
// Matches messages like "rpc error: code = Unknown desc = raft proposal dropped"
if strings.HasSuffix(ke.Message, "raft proposal dropped") {
return rt.roundTrip(req, retry-1, resp)
}
return resp, nil
}
type kubernetesError struct {
Message string `json:"message"`
Code int `json:"code"`
}
+292
View File
@@ -0,0 +1,292 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"time"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/aggregator"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/collector"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/engine"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/event"
"github.com/fluxcd/cli-utils/pkg/kstatus/polling/statusreaders"
"github.com/fluxcd/cli-utils/pkg/kstatus/status"
"github.com/fluxcd/cli-utils/pkg/kstatus/watcher"
"github.com/fluxcd/cli-utils/pkg/object"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/dynamic"
watchtools "k8s.io/client-go/tools/watch"
"helm.sh/helm/v4/internal/logging"
helmStatusReaders "helm.sh/helm/v4/internal/statusreaders"
)
type statusWaiter struct {
client dynamic.Interface
restMapper meta.RESTMapper
ctx context.Context
watchUntilReadyCtx context.Context
waitCtx context.Context
waitWithJobsCtx context.Context
waitForDeleteCtx context.Context
readers []engine.StatusReader
logging.LogHolder
}
// DefaultStatusWatcherTimeout is the timeout used by the status waiter when a
// zero timeout is provided. This prevents callers from accidentally passing a
// zero value (which would immediately cancel the context) and getting
// "context deadline exceeded" errors. SDK callers can rely on this default
// when they don't set a timeout.
var DefaultStatusWatcherTimeout = 30 * time.Second
func alwaysReady(_ *unstructured.Unstructured) (*status.Result, error) {
return &status.Result{
Status: status.CurrentStatus,
Message: "Resource is current",
}, nil
}
func (w *statusWaiter) WatchUntilReady(resourceList ResourceList, timeout time.Duration) error {
if timeout == 0 {
timeout = DefaultStatusWatcherTimeout
}
ctx, cancel := w.contextWithTimeout(w.watchUntilReadyCtx, timeout)
defer cancel()
w.Logger().Debug("waiting for resources", "count", len(resourceList), "timeout", timeout)
sw := watcher.NewDefaultStatusWatcher(w.client, w.restMapper)
jobSR := helmStatusReaders.NewCustomJobStatusReader(w.restMapper)
podSR := helmStatusReaders.NewCustomPodStatusReader(w.restMapper)
// We don't want to wait on any other resources as watchUntilReady is only for Helm hooks.
// If custom readers are defined they can be used as Helm hooks support any resource.
// We put them in front since the DelegatingStatusReader uses the first reader that matches.
genericSR := statusreaders.NewGenericStatusReader(w.restMapper, alwaysReady)
sr := &statusreaders.DelegatingStatusReader{
StatusReaders: append(w.readers, jobSR, podSR, genericSR),
}
sw.StatusReader = sr
return w.wait(ctx, resourceList, sw)
}
func (w *statusWaiter) Wait(resourceList ResourceList, timeout time.Duration) error {
if timeout == 0 {
timeout = DefaultStatusWatcherTimeout
}
ctx, cancel := w.contextWithTimeout(w.waitCtx, timeout)
defer cancel()
w.Logger().Debug("waiting for resources", "count", len(resourceList), "timeout", timeout)
sw := watcher.NewDefaultStatusWatcher(w.client, w.restMapper)
sw.StatusReader = statusreaders.NewStatusReader(w.restMapper, w.readers...)
return w.wait(ctx, resourceList, sw)
}
func (w *statusWaiter) WaitWithJobs(resourceList ResourceList, timeout time.Duration) error {
if timeout == 0 {
timeout = DefaultStatusWatcherTimeout
}
ctx, cancel := w.contextWithTimeout(w.waitWithJobsCtx, timeout)
defer cancel()
w.Logger().Debug("waiting for resources", "count", len(resourceList), "timeout", timeout)
sw := watcher.NewDefaultStatusWatcher(w.client, w.restMapper)
newCustomJobStatusReader := helmStatusReaders.NewCustomJobStatusReader(w.restMapper)
readers := append([]engine.StatusReader(nil), w.readers...)
readers = append(readers, newCustomJobStatusReader)
customSR := statusreaders.NewStatusReader(w.restMapper, readers...)
sw.StatusReader = customSR
return w.wait(ctx, resourceList, sw)
}
func (w *statusWaiter) WaitForDelete(resourceList ResourceList, timeout time.Duration) error {
if timeout == 0 {
timeout = DefaultStatusWatcherTimeout
}
ctx, cancel := w.contextWithTimeout(w.waitForDeleteCtx, timeout)
defer cancel()
w.Logger().Debug("waiting for resources to be deleted", "count", len(resourceList), "timeout", timeout)
sw := watcher.NewDefaultStatusWatcher(w.client, w.restMapper)
return w.waitForDelete(ctx, resourceList, sw)
}
func (w *statusWaiter) waitForDelete(ctx context.Context, resourceList ResourceList, sw watcher.StatusWatcher) error {
cancelCtx, cancel := context.WithCancel(ctx)
defer cancel()
resources := []object.ObjMetadata{}
for _, resource := range resourceList {
obj, err := object.RuntimeToObjMeta(resource.Object)
if err != nil {
return err
}
resources = append(resources, obj)
}
eventCh := sw.Watch(cancelCtx, resources, watcher.Options{
RESTScopeStrategy: watcher.RESTScopeNamespace,
})
statusCollector := collector.NewResourceStatusCollector(resources)
done := statusCollector.ListenWithObserver(eventCh, statusObserver(cancel, status.NotFoundStatus, w.Logger()))
<-done
if statusCollector.Error != nil {
return statusCollector.Error
}
errs := []error{}
for _, id := range resources {
rs := statusCollector.ResourceStatuses[id]
if rs.Status == status.NotFoundStatus || rs.Status == status.UnknownStatus {
continue
}
errs = append(errs, fmt.Errorf("resource %s/%s/%s still exists. status: %s, message: %s",
rs.Identifier.GroupKind.Kind, rs.Identifier.Namespace, rs.Identifier.Name, rs.Status, rs.Message))
}
if err := ctx.Err(); err != nil {
errs = append(errs, err)
}
if len(errs) > 0 {
return errors.Join(errs...)
}
return nil
}
func (w *statusWaiter) wait(ctx context.Context, resourceList ResourceList, sw watcher.StatusWatcher) error {
cancelCtx, cancel := context.WithCancel(ctx)
defer cancel()
resources := []object.ObjMetadata{}
for _, resource := range resourceList {
switch value := AsVersioned(resource).(type) {
case *appsv1.Deployment:
if value.Spec.Paused {
continue
}
}
obj, err := object.RuntimeToObjMeta(resource.Object)
if err != nil {
return err
}
resources = append(resources, obj)
}
eventCh := sw.Watch(cancelCtx, resources, watcher.Options{
RESTScopeStrategy: watcher.RESTScopeNamespace,
})
statusCollector := collector.NewResourceStatusCollector(resources)
done := statusCollector.ListenWithObserver(eventCh, statusObserver(cancel, status.CurrentStatus, w.Logger()))
<-done
if statusCollector.Error != nil {
return statusCollector.Error
}
errs := []error{}
for _, id := range resources {
rs := statusCollector.ResourceStatuses[id]
if rs.Status == status.CurrentStatus {
continue
}
errs = append(errs, fmt.Errorf("resource %s/%s/%s not ready. status: %s, message: %s",
rs.Identifier.GroupKind.Kind, rs.Identifier.Namespace, rs.Identifier.Name, rs.Status, rs.Message))
}
if err := ctx.Err(); err != nil {
errs = append(errs, err)
}
if len(errs) > 0 {
return errors.Join(errs...)
}
return nil
}
func (w *statusWaiter) contextWithTimeout(methodCtx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
if methodCtx == nil {
methodCtx = w.ctx
}
return contextWithTimeout(methodCtx, timeout)
}
func contextWithTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
if ctx == nil {
ctx = context.Background()
}
return watchtools.ContextWithOptionalTimeout(ctx, timeout)
}
func statusObserver(cancel context.CancelFunc, desired status.Status, logger *slog.Logger) collector.ObserverFunc {
return func(statusCollector *collector.ResourceStatusCollector, _ event.Event) {
var rss []*event.ResourceStatus
var nonDesiredResources []*event.ResourceStatus
for _, rs := range statusCollector.ResourceStatuses {
if rs == nil {
continue
}
// If a resource is already deleted before waiting has started, it will show as unknown.
// This check ensures we don't wait forever for a resource that is already deleted.
if rs.Status == status.UnknownStatus && desired == status.NotFoundStatus {
continue
}
// Failed is a terminal state. This check ensures we don't wait forever for a resource
// that has already failed, as intervention is required to resolve the failure.
if rs.Status == status.FailedStatus && desired == status.CurrentStatus {
continue
}
rss = append(rss, rs)
if rs.Status != desired {
nonDesiredResources = append(nonDesiredResources, rs)
}
}
if aggregator.AggregateStatus(rss, desired) == desired {
logger.Debug("all resources achieved desired status", "desiredStatus", desired, "resourceCount", len(rss))
cancel()
return
}
if len(nonDesiredResources) > 0 {
// Log a single resource so the user knows what they're waiting for without an overwhelming amount of output
sort.Slice(nonDesiredResources, func(i, j int) bool {
return nonDesiredResources[i].Identifier.Name < nonDesiredResources[j].Identifier.Name
})
first := nonDesiredResources[0]
logger.Debug("waiting for resource", "namespace", first.Identifier.Namespace, "name", first.Identifier.Name, "kind", first.Identifier.GroupKind.Kind, "expectedStatus", desired, "actualStatus", first.Status)
}
}
}
type hookOnlyWaiter struct {
sw *statusWaiter
}
func (w *hookOnlyWaiter) WatchUntilReady(resourceList ResourceList, timeout time.Duration) error {
return w.sw.WatchUntilReady(resourceList, timeout)
}
func (w *hookOnlyWaiter) Wait(_ ResourceList, _ time.Duration) error {
return nil
}
func (w *hookOnlyWaiter) WaitWithJobs(_ ResourceList, _ time.Duration) error {
return nil
}
func (w *hookOnlyWaiter) WaitForDelete(_ ResourceList, _ time.Duration) error {
return nil
}
+345
View File
@@ -0,0 +1,345 @@
/*
Copyright The Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube // import "helm.sh/helm/v4/pkg/kube"
import (
"context"
"fmt"
"log/slog"
"net/http"
"time"
appsv1 "k8s.io/api/apps/v1"
appsv1beta1 "k8s.io/api/apps/v1beta1"
appsv1beta2 "k8s.io/api/apps/v1beta2"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/kubernetes"
cachetools "k8s.io/client-go/tools/cache"
watchtools "k8s.io/client-go/tools/watch"
"k8s.io/apimachinery/pkg/util/wait"
)
// legacyWaiter is the legacy implementation of the Waiter interface. This logic was used by default in Helm 3
// Helm 4 now uses the StatusWaiter implementation instead
type legacyWaiter struct {
c ReadyChecker
kubeClient *kubernetes.Clientset
ctx context.Context
}
func (hw *legacyWaiter) Wait(resources ResourceList, timeout time.Duration) error {
hw.c = NewReadyChecker(hw.kubeClient, PausedAsReady(true))
return hw.waitForResources(resources, timeout)
}
func (hw *legacyWaiter) WaitWithJobs(resources ResourceList, timeout time.Duration) error {
hw.c = NewReadyChecker(hw.kubeClient, PausedAsReady(true), CheckJobs(true))
return hw.waitForResources(resources, timeout)
}
// waitForResources polls to get the current status of all pods, PVCs, Services and
// Jobs(optional) until all are ready or a timeout is reached
func (hw *legacyWaiter) waitForResources(created ResourceList, timeout time.Duration) error {
slog.Debug("beginning wait for resources", "count", len(created), "timeout", timeout)
ctx, cancel := hw.contextWithTimeout(timeout)
defer cancel()
numberOfErrors := make([]int, len(created))
for i := range numberOfErrors {
numberOfErrors[i] = 0
}
return wait.PollUntilContextCancel(ctx, 2*time.Second, true, func(ctx context.Context) (bool, error) {
waitRetries := 30
for i, v := range created {
ready, err := hw.c.IsReady(ctx, v)
if waitRetries > 0 && hw.isRetryableError(err, v) {
numberOfErrors[i]++
if numberOfErrors[i] > waitRetries {
slog.Debug("max number of retries reached", "resource", v.Name, "retries", numberOfErrors[i])
return false, err
}
slog.Debug("retrying resource readiness", "resource", v.Name, "currentRetries", numberOfErrors[i]-1, "maxRetries", waitRetries)
return false, nil
}
numberOfErrors[i] = 0
if !ready {
return false, err
}
}
return true, nil
})
}
func (hw *legacyWaiter) isRetryableError(err error, resource *resource.Info) bool {
if err == nil {
return false
}
slog.Debug(
"error received when checking resource status",
slog.String("resource", resource.Name),
slog.Any("error", err),
)
if ev, ok := err.(*apierrors.StatusError); ok {
statusCode := ev.Status().Code
retryable := hw.isRetryableHTTPStatusCode(statusCode)
slog.Debug(
"status code received",
slog.String("resource", resource.Name),
slog.Int("statusCode", int(statusCode)),
slog.Bool("retryable", retryable),
)
return retryable
}
slog.Debug("retryable error assumed", "resource", resource.Name)
return true
}
func (hw *legacyWaiter) isRetryableHTTPStatusCode(httpStatusCode int32) bool {
return httpStatusCode == 0 || httpStatusCode == http.StatusTooManyRequests || (httpStatusCode >= 500 && httpStatusCode != http.StatusNotImplemented)
}
// WaitForDelete polls to check if all the resources are deleted or a timeout is reached
func (hw *legacyWaiter) WaitForDelete(deleted ResourceList, timeout time.Duration) error {
slog.Debug("beginning wait for resources to be deleted", "count", len(deleted), "timeout", timeout)
startTime := time.Now()
ctx, cancel := hw.contextWithTimeout(timeout)
defer cancel()
err := wait.PollUntilContextCancel(ctx, 2*time.Second, true, func(_ context.Context) (bool, error) {
for _, v := range deleted {
err := v.Get()
if err == nil || !apierrors.IsNotFound(err) {
return false, err
}
}
return true, nil
})
elapsed := time.Since(startTime).Round(time.Second)
if err != nil {
slog.Debug("wait for resources failed", slog.Duration("elapsed", elapsed), slog.Any("error", err))
} else {
slog.Debug("wait for resources succeeded", slog.Duration("elapsed", elapsed))
}
return err
}
// SelectorsForObject returns the pod label selector for a given object
//
// Modified version of https://github.com/kubernetes/kubernetes/blob/v1.14.1/pkg/kubectl/polymorphichelpers/helpers.go#L84
func SelectorsForObject(object runtime.Object) (selector labels.Selector, err error) {
switch t := object.(type) {
case *extensionsv1beta1.ReplicaSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1.ReplicaSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta2.ReplicaSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *corev1.ReplicationController:
selector = labels.SelectorFromSet(t.Spec.Selector)
case *appsv1.StatefulSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta1.StatefulSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta2.StatefulSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *extensionsv1beta1.DaemonSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1.DaemonSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta2.DaemonSet:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *extensionsv1beta1.Deployment:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1.Deployment:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta1.Deployment:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *appsv1beta2.Deployment:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *batchv1.Job:
selector, err = metav1.LabelSelectorAsSelector(t.Spec.Selector)
case *corev1.Service:
if len(t.Spec.Selector) == 0 {
return nil, fmt.Errorf("invalid service '%s': Service is defined without a selector", t.Name)
}
selector = labels.SelectorFromSet(t.Spec.Selector)
default:
return nil, fmt.Errorf("selector for %T not implemented", object)
}
if err != nil {
return selector, fmt.Errorf("invalid label selector: %w", err)
}
return selector, nil
}
func (hw *legacyWaiter) watchTimeout(t time.Duration) func(*resource.Info) error {
return func(info *resource.Info) error {
return hw.watchUntilReady(t, info)
}
}
// WatchUntilReady watches the resources given and waits until it is ready.
//
// This method is mainly for hook implementations. It watches for a resource to
// hit a particular milestone. The milestone depends on the Kind.
//
// For most kinds, it checks to see if the resource is marked as Added or Modified
// by the Kubernetes event stream. For some kinds, it does more:
//
// - Jobs: A job is marked "Ready" when it has successfully completed. This is
// ascertained by watching the Status fields in a job's output.
// - Pods: A pod is marked "Ready" when it has successfully completed. This is
// ascertained by watching the status.phase field in a pod's output.
//
// Handling for other kinds will be added as necessary.
func (hw *legacyWaiter) WatchUntilReady(resources ResourceList, timeout time.Duration) error {
// For jobs, there's also the option to do poll c.Jobs(namespace).Get():
// https://github.com/adamreese/kubernetes/blob/master/test/e2e/job.go#L291-L300
return perform(resources, hw.watchTimeout(timeout))
}
func (hw *legacyWaiter) watchUntilReady(timeout time.Duration, info *resource.Info) error {
kind := info.Mapping.GroupVersionKind.Kind
switch kind {
case "Job", "Pod":
default:
return nil
}
slog.Debug("watching for resource changes", "kind", kind, "resource", info.Name, "timeout", timeout)
// Use a selector on the name of the resource. This should be unique for the
// given version and kind
selector, err := fields.ParseSelector(fmt.Sprintf("metadata.name=%s", info.Name))
if err != nil {
return err
}
lw := cachetools.NewListWatchFromClient(info.Client, info.Mapping.Resource.Resource, info.Namespace, selector)
// What we watch for depends on the Kind.
// - For a Job, we watch for completion.
// - For all else, we watch until Ready.
// In the future, we might want to add some special logic for types
// like Ingress, Volume, etc.
ctx, cancel := hw.contextWithTimeout(timeout)
defer cancel()
_, err = watchtools.UntilWithSync(ctx, lw, &unstructured.Unstructured{}, nil, func(e watch.Event) (bool, error) {
// Make sure the incoming object is versioned as we use unstructured
// objects when we build manifests
obj := convertWithMapper(e.Object, info.Mapping)
switch e.Type {
case watch.Added, watch.Modified:
// For things like a secret or a config map, this is the best indicator
// we get. We care mostly about jobs, where what we want to see is
// the status go into a good state. For other types, like ReplicaSet
// we don't really do anything to support these as hooks.
slog.Debug("add/modify event received", "resource", info.Name, "eventType", e.Type)
switch kind {
case "Job":
return hw.waitForJob(obj, info.Name)
case "Pod":
return hw.waitForPodSuccess(obj, info.Name)
}
return true, nil
case watch.Deleted:
slog.Debug("deleted event received", "resource", info.Name)
return true, nil
case watch.Error:
// Handle error and return with an error.
slog.Error("error event received", "resource", info.Name)
return true, fmt.Errorf("failed to deploy %s", info.Name)
default:
return false, nil
}
})
return err
}
// waitForJob is a helper that waits for a job to complete.
//
// This operates on an event returned from a watcher.
func (hw *legacyWaiter) waitForJob(obj runtime.Object, name string) (bool, error) {
o, ok := obj.(*batchv1.Job)
if !ok {
return true, fmt.Errorf("expected %s to be a *batch.Job, got %T", name, obj)
}
for _, c := range o.Status.Conditions {
if c.Type == batchv1.JobComplete && c.Status == "True" {
return true, nil
} else if c.Type == batchv1.JobFailed && c.Status == "True" {
slog.Error("job failed", "job", name, "reason", c.Reason)
return true, fmt.Errorf("job %s failed: %s", name, c.Reason)
}
}
slog.Debug("job status update", "job", name, "active", o.Status.Active, "failed", o.Status.Failed, "succeeded", o.Status.Succeeded)
return false, nil
}
// waitForPodSuccess is a helper that waits for a pod to complete.
//
// This operates on an event returned from a watcher.
func (hw *legacyWaiter) waitForPodSuccess(obj runtime.Object, name string) (bool, error) {
o, ok := obj.(*corev1.Pod)
if !ok {
return true, fmt.Errorf("expected %s to be a *v1.Pod, got %T", name, obj)
}
switch o.Status.Phase {
case corev1.PodSucceeded:
slog.Debug("pod succeeded", "pod", o.Name)
return true, nil
case corev1.PodFailed:
slog.Error("pod failed", "pod", o.Name)
return true, fmt.Errorf("pod %s failed", o.Name)
case corev1.PodPending:
slog.Debug("pod pending", "pod", o.Name)
case corev1.PodRunning:
slog.Debug("pod running", "pod", o.Name)
case corev1.PodUnknown:
slog.Debug("pod unknown", "pod", o.Name)
}
return false, nil
}
func (hw *legacyWaiter) contextWithTimeout(timeout time.Duration) (context.Context, context.CancelFunc) {
return contextWithTimeout(hw.ctx, timeout)
}