Skip to content

Commit

Permalink
Extract out common parse-package logic (#3711)
Browse files Browse the repository at this point in the history
We had this code duplicated in a few places also.
  • Loading branch information
justinsb committed Jan 5, 2023
1 parent dc931ec commit 3d80f24
Show file tree
Hide file tree
Showing 6 changed files with 192 additions and 141 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,11 @@ import (
"context"
"flag"
"fmt"
"path/filepath"
"strings"
"unicode"

api "github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/applyset"
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -34,7 +33,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/yaml"
)

type Options struct {
Expand Down Expand Up @@ -98,15 +96,6 @@ func (r *KlippyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
return ctrl.Result{}, nil
}

func isWhitespace(s string) bool {
for _, r := range s {
if !unicode.IsSpace(r) {
return false
}
}
return true
}

func (r *KlippyReconciler) reconcile(ctx context.Context, parent *api.PackageRevision, parentResources *api.PackageRevisionResources) error {
log := log.FromContext(ctx)

Expand All @@ -121,7 +110,7 @@ func (r *KlippyReconciler) reconcile(ctx context.Context, parent *api.PackageRev
return err
}

parentObjects, err := parseObjects(ctx, parentResources)
parentObjects, err := objects.Parser{}.AsUnstructureds(parentResources.Spec.Resources)
if err != nil {
return err
}
Expand Down Expand Up @@ -166,10 +155,11 @@ func (r *KlippyReconciler) parseBlueprints(ctx context.Context, packageRevisions
return nil, fmt.Errorf("error fetching PackageRevisionResources %v: %w", id, err)
}

objects, err := parseObjects(ctx, &packageRevisionResources)
objects, err := objects.Parser{}.AsUnstructureds(packageRevisionResources.Spec.Resources)
if err != nil {
return nil, err
}

blueprint := &blueprint{
Objects: objects,
ID: id,
Expand All @@ -180,41 +170,6 @@ func (r *KlippyReconciler) parseBlueprints(ctx context.Context, packageRevisions
return blueprints, nil
}

func parseObjects(ctx context.Context, subject *api.PackageRevisionResources) ([]*unstructured.Unstructured, error) {
log := log.FromContext(ctx)

// TODO: Does this function exist somewhere?
var objects []*unstructured.Unstructured

for path, contents := range subject.Spec.Resources {
ext := filepath.Ext(path)
ext = strings.ToLower(ext)

switch ext {
case ".yaml", ".yml":
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
for _, s := range strings.Split(contents, "\n---\n") {
if isWhitespace(s) {
continue
}

o := &unstructured.Unstructured{}
if err := yaml.Unmarshal([]byte(s), o); err != nil {
return nil, fmt.Errorf("error parsing package as kubernetes object: %w", err)
}

// TODO: sync with kpt logic; skip objects marked with the local-only annotation
objects = append(objects, o)
}

default:
log.V(2).Info("skipping file with unhandled extension", "path", path)
}
}

return objects, nil
}

type bindingSlotRequirements struct {
MatchLabels map[string]string
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,15 @@ import (
"flag"
"fmt"
"os"
"path"
"strings"
"unicode"

kptoci "github.com/GoogleContainerTools/kpt/pkg/oci"
api "github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/api/v1alpha1"
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/applyset"
"github.com/GoogleContainerTools/kpt/porch/controllers/remoterootsyncsets/pkg/remoteclient"
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
"github.com/GoogleContainerTools/kpt/porch/pkg/oci"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/discovery"
memory "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/dynamic"
Expand All @@ -40,7 +37,6 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/yaml"
)

var (
Expand Down Expand Up @@ -325,50 +321,16 @@ func (r *RemoteRootSyncSetReconciler) BuildObjectsToApply(ctx context.Context, s
return nil, err
}

var objects []applyset.ApplyableObject

for filePath, fileContents := range resources.Contents {
ext := path.Ext(filePath)
ext = strings.ToLower(ext)

parse := false
switch ext {
case ".yaml", ".yml":
parse = true

default:
klog.Warningf("ignoring non-yaml file %s", filePath)
}

if !parse {
continue
}
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
for _, s := range strings.Split(fileContents, "\n---\n") {
if isWhitespace(s) {
continue
}

o := &unstructured.Unstructured{}
if err := yaml.Unmarshal([]byte(s), &o); err != nil {
return nil, fmt.Errorf("error parsing yaml from %s: %w", filePath, err)
}

// TODO: sync with kpt logic; skip objects marked with the local-only annotation
objects = append(objects, o)
}
unstructureds, err := objects.Parser{}.AsUnstructureds(resources.Contents)
if err != nil {
return nil, err
}

return objects, nil
}

func isWhitespace(s string) bool {
for _, r := range s {
if !unicode.IsSpace(r) {
return false
}
var applyables []applyset.ApplyableObject
for _, u := range unstructureds {
applyables = append(applyables, u)
}
return true
return applyables, nil
}

// SetupWithManager sets up the controller with the Manager.
Expand Down
57 changes: 11 additions & 46 deletions porch/pkg/engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,9 @@ import (
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"reflect"
"strings"
"unicode"

"github.com/GoogleContainerTools/kpt/internal/builtins"
"github.com/GoogleContainerTools/kpt/internal/fnruntime"
Expand All @@ -35,6 +33,7 @@ import (
"github.com/GoogleContainerTools/kpt/porch/pkg/cache"
"github.com/GoogleContainerTools/kpt/porch/pkg/kpt"
"github.com/GoogleContainerTools/kpt/porch/pkg/meta"
"github.com/GoogleContainerTools/kpt/porch/pkg/objects"
"github.com/GoogleContainerTools/kpt/porch/pkg/repository"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
Expand Down Expand Up @@ -1385,42 +1384,17 @@ func (cad *cadEngine) recloneAndReplay(ctx context.Context, repo repository.Repo

// ExtractContextConfigMap returns the package-context configmap, if found
func ExtractContextConfigMap(resources map[string]string) (*unstructured.Unstructured, error) {
var matches []*unstructured.Unstructured

for itemPath, itemContents := range resources {
ext := path.Ext(itemPath)
ext = strings.ToLower(ext)

parse := false
switch ext {
case ".yaml", ".yml":
parse = true

default:
klog.Warningf("ignoring non-yaml file %s", itemPath)
}

if !parse {
continue
}
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
for _, s := range strings.Split(itemContents, "\n---\n") {
if isWhitespace(s) {
continue
}

o := &unstructured.Unstructured{}
if err := yaml.Unmarshal([]byte(s), &o); err != nil {
return nil, fmt.Errorf("error parsing yaml from %s: %w", itemPath, err)
}

// TODO: sync with kpt logic; skip objects marked with the local-only annotation
unstructureds, err := objects.Parser{}.AsUnstructureds(resources)
if err != nil {
return nil, err
}

configMapGK := schema.GroupKind{Kind: "ConfigMap"}
if o.GroupVersionKind().GroupKind() == configMapGK {
if o.GetName() == builtins.PkgContextName {
matches = append(matches, o)
}
var matches []*unstructured.Unstructured
for _, o := range unstructureds {
configMapGK := schema.GroupKind{Kind: "ConfigMap"}
if o.GroupVersionKind().GroupKind() == configMapGK {
if o.GetName() == builtins.PkgContextName {
matches = append(matches, o)
}
}
}
Expand All @@ -1434,12 +1408,3 @@ func ExtractContextConfigMap(resources map[string]string) (*unstructured.Unstruc

return matches[0], nil
}

func isWhitespace(s string) bool {
for _, r := range s {
if !unicode.IsSpace(r) {
return false
}
}
return true
}
105 changes: 105 additions & 0 deletions porch/pkg/objects/extract.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package objects

import (
"fmt"
"path"
"strings"
"unicode"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/klog/v2"
"sigs.k8s.io/yaml"
)

type Parser struct {
IncludeLocalConfig bool
}

func (p Parser) AsUnstructureds(resources map[string]string) ([]*unstructured.Unstructured, error) {
objectList, err := p.AsObjectList(resources)
if err != nil {
return nil, err
}
unstructureds, err := objectList.AsUnstructured()
if err != nil {
return nil, err
}
return unstructureds, nil
}

func (p Parser) AsObjectList(resources map[string]string) (*ObjectList, error) {
var items []*Object

for filePath, fileContents := range resources {
ext := path.Ext(filePath)
ext = strings.ToLower(ext)

parse := false
switch ext {
case ".yaml", ".yml":
parse = true

default:
klog.Warningf("ignoring non-yaml file %s", filePath)
}

if !parse {
continue
}
// TODO: Use https://github.com/kubernetes-sigs/kustomize/blob/a5b61016bb40c30dd1b0a78290b28b2330a0383e/kyaml/kio/byteio_reader.go#L170 or similar?
for _, s := range strings.Split(fileContents, "\n---\n") {
if isWhitespace(s) {
continue
}

raw := []byte(s)
u := &unstructured.Unstructured{}
if err := yaml.Unmarshal(raw, &u); err != nil {
return nil, fmt.Errorf("error parsing yaml from %s: %w", filePath, err)
}

annotations := u.GetAnnotations()

if !p.IncludeLocalConfig {
s := annotations["config.kubernetes.io/local-config"]
if s == "true" {
continue
}
}

obj := &Object{
filePath: filePath,
raw: raw,
u: u,
}

// TODO: sync with kpt logic; skip objects marked with the local-only annotation
items = append(items, obj)
}
}

return &ObjectList{Items: items}, nil
}

func isWhitespace(s string) bool {
for _, r := range s {
if !unicode.IsSpace(r) {
return false
}
}
return true
}

0 comments on commit 3d80f24

Please sign in to comment.