working commit
This commit is contained in:
+447
@@ -0,0 +1,447 @@
|
||||
// Copyright 2019 The Kubernetes Authors.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package kio contains low-level libraries for reading, modifying and writing
|
||||
// Resource Configuration and packages.
|
||||
package kio
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"sigs.k8s.io/kustomize/kyaml/errors"
|
||||
"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
|
||||
"sigs.k8s.io/kustomize/kyaml/yaml"
|
||||
)
|
||||
|
||||
// Reader reads ResourceNodes. Analogous to io.Reader.
|
||||
type Reader interface {
|
||||
Read() ([]*yaml.RNode, error)
|
||||
}
|
||||
|
||||
// ResourceNodeSlice is a collection of ResourceNodes.
|
||||
// While ResourceNodeSlice has no inherent constraints on ordering or uniqueness, specific
|
||||
// Readers, Filters or Writers may have constraints.
|
||||
type ResourceNodeSlice []*yaml.RNode
|
||||
|
||||
var _ Reader = ResourceNodeSlice{}
|
||||
|
||||
func (o ResourceNodeSlice) Read() ([]*yaml.RNode, error) {
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Writer writes ResourceNodes. Analogous to io.Writer.
|
||||
type Writer interface {
|
||||
Write([]*yaml.RNode) error
|
||||
}
|
||||
|
||||
// WriterFunc implements a Writer as a function.
|
||||
type WriterFunc func([]*yaml.RNode) error
|
||||
|
||||
func (fn WriterFunc) Write(o []*yaml.RNode) error {
|
||||
return fn(o)
|
||||
}
|
||||
|
||||
// ReaderWriter implements both Reader and Writer interfaces
|
||||
type ReaderWriter interface {
|
||||
Reader
|
||||
Writer
|
||||
}
|
||||
|
||||
// Filter modifies a collection of Resource Configuration by returning the modified slice.
|
||||
// When possible, Filters should be serializable to yaml so that they can be described
|
||||
// as either data or code.
|
||||
//
|
||||
// Analogous to http://www.linfo.org/filters.html
|
||||
type Filter interface {
|
||||
Filter([]*yaml.RNode) ([]*yaml.RNode, error)
|
||||
}
|
||||
|
||||
// TrackableFilter is an extension of Filter which is also capable of tracking
|
||||
// which fields were mutated by the filter.
|
||||
type TrackableFilter interface {
|
||||
Filter
|
||||
WithMutationTracker(func(key, value, tag string, node *yaml.RNode))
|
||||
}
|
||||
|
||||
// FilterFunc implements a Filter as a function.
|
||||
type FilterFunc func([]*yaml.RNode) ([]*yaml.RNode, error)
|
||||
|
||||
func (fn FilterFunc) Filter(o []*yaml.RNode) ([]*yaml.RNode, error) {
|
||||
return fn(o)
|
||||
}
|
||||
|
||||
// Pipeline reads Resource Configuration from a set of Inputs, applies some
|
||||
// transformation filters, and writes the results to a set of Outputs.
|
||||
//
|
||||
// Analogous to http://www.linfo.org/pipes.html
|
||||
type Pipeline struct {
|
||||
// Inputs provide sources for Resource Configuration to be read.
|
||||
Inputs []Reader `yaml:"inputs,omitempty"`
|
||||
|
||||
// Filters are transformations applied to the Resource Configuration.
|
||||
// They are applied in the order they are specified.
|
||||
// Analogous to http://www.linfo.org/filters.html
|
||||
Filters []Filter `yaml:"filters,omitempty"`
|
||||
|
||||
// Outputs are where the transformed Resource Configuration is written.
|
||||
Outputs []Writer `yaml:"outputs,omitempty"`
|
||||
|
||||
// ContinueOnEmptyResult configures what happens when a filter in the pipeline
|
||||
// returns an empty result.
|
||||
// If it is false (default), subsequent filters will be skipped and the result
|
||||
// will be returned immediately. This is useful as an optimization when you
|
||||
// know that subsequent filters will not alter the empty result.
|
||||
// If it is true, the empty result will be provided as input to the next
|
||||
// filter in the list. This is useful when subsequent functions in the
|
||||
// pipeline may generate new resources.
|
||||
ContinueOnEmptyResult bool `yaml:"continueOnEmptyResult,omitempty"`
|
||||
}
|
||||
|
||||
// Execute executes each step in the sequence, returning immediately after encountering
|
||||
// any error as part of the Pipeline.
|
||||
func (p Pipeline) Execute() error {
|
||||
return p.ExecuteWithCallback(nil)
|
||||
}
|
||||
|
||||
// PipelineExecuteCallbackFunc defines a callback function that will be called each time a step in the pipeline succeeds.
|
||||
type PipelineExecuteCallbackFunc = func(op Filter)
|
||||
|
||||
// ExecuteWithCallback executes each step in the sequence, returning immediately after encountering
|
||||
// any error as part of the Pipeline. The callback will be called each time a step succeeds.
|
||||
func (p Pipeline) ExecuteWithCallback(callback PipelineExecuteCallbackFunc) error {
|
||||
var result []*yaml.RNode
|
||||
|
||||
// read from the inputs
|
||||
for _, i := range p.Inputs {
|
||||
nodes, err := i.Read()
|
||||
if err != nil {
|
||||
return errors.Wrap(err)
|
||||
}
|
||||
result = append(result, nodes...)
|
||||
}
|
||||
|
||||
// apply operations
|
||||
for i := range p.Filters {
|
||||
// Not all RNodes passed through kio.Pipeline have metadata nor should
|
||||
// they all be required to.
|
||||
nodeAnnos, err := PreprocessResourcesForInternalAnnotationMigration(result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
op := p.Filters[i]
|
||||
if callback != nil {
|
||||
callback(op)
|
||||
}
|
||||
result, err = op.Filter(result)
|
||||
// TODO (issue 2872): This len(result) == 0 should be removed and empty result list should be
|
||||
// handled by outputs. However currently some writer like LocalPackageReadWriter
|
||||
// will clear the output directory and which will cause unpredictable results
|
||||
if len(result) == 0 && !p.ContinueOnEmptyResult || err != nil {
|
||||
return errors.Wrap(err)
|
||||
}
|
||||
|
||||
// If either the internal annotations for path, index, and id OR the legacy
|
||||
// annotations for path, index, and id are changed, we have to update the other.
|
||||
err = ReconcileInternalAnnotations(result, nodeAnnos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// write to the outputs
|
||||
for _, o := range p.Outputs {
|
||||
if err := o.Write(result); err != nil {
|
||||
return errors.Wrap(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FilterAll runs the yaml.Filter against all inputs
|
||||
func FilterAll(filter yaml.Filter) Filter {
|
||||
return FilterFunc(func(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
|
||||
for i := range nodes {
|
||||
_, err := filter.Filter(nodes[i])
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err)
|
||||
}
|
||||
}
|
||||
return nodes, nil
|
||||
})
|
||||
}
|
||||
|
||||
// PreprocessResourcesForInternalAnnotationMigration returns a mapping from id to all
|
||||
// internal annotations, so that we can use it to reconcile the annotations
|
||||
// later. This is necessary because currently both internal-prefixed annotations
|
||||
// and legacy annotations are currently supported, and a change to one must be
|
||||
// reflected in the other if needed.
|
||||
func PreprocessResourcesForInternalAnnotationMigration(result []*yaml.RNode) (map[string]map[string]string, error) {
|
||||
idToAnnosMap := make(map[string]map[string]string)
|
||||
for i := range result {
|
||||
idStr := strconv.Itoa(i)
|
||||
err := result[i].PipeE(yaml.SetAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation, idStr))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idToAnnosMap[idStr] = kioutil.GetInternalAnnotations(result[i])
|
||||
if err = kioutil.CopyLegacyAnnotations(result[i]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, _ := result[i].GetMeta()
|
||||
if err = checkMismatchedAnnos(meta.Annotations); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return idToAnnosMap, nil
|
||||
}
|
||||
|
||||
func checkMismatchedAnnos(annotations map[string]string) error {
|
||||
path := annotations[kioutil.PathAnnotation]
|
||||
index := annotations[kioutil.IndexAnnotation]
|
||||
id := annotations[kioutil.IdAnnotation]
|
||||
|
||||
legacyPath := annotations[kioutil.LegacyPathAnnotation]
|
||||
legacyIndex := annotations[kioutil.LegacyIndexAnnotation]
|
||||
legacyId := annotations[kioutil.LegacyIdAnnotation]
|
||||
|
||||
// if prior to running the functions, the legacy and internal annotations differ,
|
||||
// throw an error as we cannot infer the user's intent.
|
||||
if path != "" && legacyPath != "" && path != legacyPath {
|
||||
return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
|
||||
}
|
||||
if index != "" && legacyIndex != "" && index != legacyIndex {
|
||||
return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
|
||||
}
|
||||
if id != "" && legacyId != "" && id != legacyId {
|
||||
return fmt.Errorf("resource input to function has mismatched legacy and internal id annotations")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type nodeAnnotations struct {
|
||||
path string
|
||||
index string
|
||||
id string
|
||||
}
|
||||
|
||||
// ReconcileInternalAnnotations reconciles the annotation format for path, index and id annotations.
|
||||
// It will ensure the output annotation format matches the format in the input. e.g. if the input
|
||||
// format uses the legacy format and the output will be converted to the legacy format if it's not.
|
||||
func ReconcileInternalAnnotations(result []*yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
|
||||
useInternal, useLegacy, err := determineAnnotationsFormat(nodeAnnosMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := range result {
|
||||
// if only one annotation is set, set the other.
|
||||
err = missingInternalOrLegacyAnnotations(result[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// we must check to see if the function changed either the new internal annotations
|
||||
// or the old legacy annotations. If one is changed, the change must be reflected
|
||||
// in the other.
|
||||
err = checkAnnotationsAltered(result[i], nodeAnnosMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// We invoke determineAnnotationsFormat to find out if the original annotations
|
||||
// use the internal or (and) the legacy format. We format the resources to
|
||||
// make them consistent with original format.
|
||||
err = formatInternalAnnotations(result[i], useInternal, useLegacy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// if the annotations are still somehow out of sync, throw an error
|
||||
meta, _ := result[i].GetMeta()
|
||||
err = checkMismatchedAnnos(meta.Annotations)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = result[i].Pipe(yaml.ClearAnnotation(kioutil.InternalAnnotationsMigrationResourceIDAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// determineAnnotationsFormat determines if the resources are using one of the internal and legacy annotation format or both of them.
|
||||
func determineAnnotationsFormat(nodeAnnosMap map[string]map[string]string) (bool, bool, error) {
|
||||
var useInternal, useLegacy bool
|
||||
var err error
|
||||
|
||||
if len(nodeAnnosMap) == 0 {
|
||||
return true, true, nil
|
||||
}
|
||||
|
||||
var internal, legacy *bool
|
||||
for _, annos := range nodeAnnosMap {
|
||||
_, foundPath := annos[kioutil.PathAnnotation]
|
||||
_, foundIndex := annos[kioutil.IndexAnnotation]
|
||||
_, foundId := annos[kioutil.IdAnnotation]
|
||||
_, foundLegacyPath := annos[kioutil.LegacyPathAnnotation]
|
||||
_, foundLegacyIndex := annos[kioutil.LegacyIndexAnnotation]
|
||||
_, foundLegacyId := annos[kioutil.LegacyIdAnnotation]
|
||||
|
||||
if !(foundPath || foundIndex || foundId || foundLegacyPath || foundLegacyIndex || foundLegacyId) {
|
||||
continue
|
||||
}
|
||||
|
||||
foundOneOf := foundPath || foundIndex || foundId
|
||||
if internal == nil {
|
||||
f := foundOneOf
|
||||
internal = &f
|
||||
}
|
||||
if (foundOneOf && !*internal) || (!foundOneOf && *internal) {
|
||||
err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
|
||||
return useInternal, useLegacy, err
|
||||
}
|
||||
|
||||
foundOneOf = foundLegacyPath || foundLegacyIndex || foundLegacyId
|
||||
if legacy == nil {
|
||||
f := foundOneOf
|
||||
legacy = &f
|
||||
}
|
||||
if (foundOneOf && !*legacy) || (!foundOneOf && *legacy) {
|
||||
err = fmt.Errorf("the annotation formatting in the input resources is not consistent")
|
||||
return useInternal, useLegacy, err
|
||||
}
|
||||
}
|
||||
if internal != nil {
|
||||
useInternal = *internal
|
||||
}
|
||||
if legacy != nil {
|
||||
useLegacy = *legacy
|
||||
}
|
||||
return useInternal, useLegacy, err
|
||||
}
|
||||
|
||||
func missingInternalOrLegacyAnnotations(rn *yaml.RNode) error {
|
||||
if err := missingInternalOrLegacyAnnotation(rn, kioutil.PathAnnotation, kioutil.LegacyPathAnnotation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := missingInternalOrLegacyAnnotation(rn, kioutil.IndexAnnotation, kioutil.LegacyIndexAnnotation); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := missingInternalOrLegacyAnnotation(rn, kioutil.IdAnnotation, kioutil.LegacyIdAnnotation); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func missingInternalOrLegacyAnnotation(rn *yaml.RNode, newKey string, legacyKey string) error {
|
||||
meta, _ := rn.GetMeta()
|
||||
annotations := meta.Annotations
|
||||
value := annotations[newKey]
|
||||
legacyValue := annotations[legacyKey]
|
||||
|
||||
if value == "" && legacyValue == "" {
|
||||
// do nothing
|
||||
return nil
|
||||
}
|
||||
|
||||
if value == "" {
|
||||
// new key is not set, copy from legacy key
|
||||
if err := rn.PipeE(yaml.SetAnnotation(newKey, legacyValue)); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if legacyValue == "" {
|
||||
// legacy key is not set, copy from new key
|
||||
if err := rn.PipeE(yaml.SetAnnotation(legacyKey, value)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkAnnotationsAltered(rn *yaml.RNode, nodeAnnosMap map[string]map[string]string) error {
|
||||
meta, _ := rn.GetMeta()
|
||||
annotations := meta.Annotations
|
||||
// get the resource's current path, index, and ids from the new annotations
|
||||
internal := nodeAnnotations{
|
||||
path: annotations[kioutil.PathAnnotation],
|
||||
index: annotations[kioutil.IndexAnnotation],
|
||||
id: annotations[kioutil.IdAnnotation],
|
||||
}
|
||||
|
||||
// get the resource's current path, index, and ids from the legacy annotations
|
||||
legacy := nodeAnnotations{
|
||||
path: annotations[kioutil.LegacyPathAnnotation],
|
||||
index: annotations[kioutil.LegacyIndexAnnotation],
|
||||
id: annotations[kioutil.LegacyIdAnnotation],
|
||||
}
|
||||
|
||||
rid := annotations[kioutil.InternalAnnotationsMigrationResourceIDAnnotation]
|
||||
originalAnnotations, found := nodeAnnosMap[rid]
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
originalPath, found := originalAnnotations[kioutil.PathAnnotation]
|
||||
if !found {
|
||||
originalPath = originalAnnotations[kioutil.LegacyPathAnnotation]
|
||||
}
|
||||
if originalPath != "" {
|
||||
switch {
|
||||
case originalPath != internal.path && originalPath != legacy.path && internal.path != legacy.path:
|
||||
return fmt.Errorf("resource input to function has mismatched legacy and internal path annotations")
|
||||
case originalPath != internal.path:
|
||||
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyPathAnnotation, internal.path)); err != nil {
|
||||
return err
|
||||
}
|
||||
case originalPath != legacy.path:
|
||||
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, legacy.path)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
originalIndex, found := originalAnnotations[kioutil.IndexAnnotation]
|
||||
if !found {
|
||||
originalIndex = originalAnnotations[kioutil.LegacyIndexAnnotation]
|
||||
}
|
||||
if originalIndex != "" {
|
||||
switch {
|
||||
case originalIndex != internal.index && originalIndex != legacy.index && internal.index != legacy.index:
|
||||
return fmt.Errorf("resource input to function has mismatched legacy and internal index annotations")
|
||||
case originalIndex != internal.index:
|
||||
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.LegacyIndexAnnotation, internal.index)); err != nil {
|
||||
return err
|
||||
}
|
||||
case originalIndex != legacy.index:
|
||||
if _, err := rn.Pipe(yaml.SetAnnotation(kioutil.IndexAnnotation, legacy.index)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatInternalAnnotations(rn *yaml.RNode, useInternal, useLegacy bool) error {
|
||||
if !useInternal {
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IdAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.PathAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.IndexAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !useLegacy {
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIdAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyPathAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rn.PipeE(yaml.ClearAnnotation(kioutil.LegacyIndexAnnotation)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user