Skip to content
This repository has been archived by the owner on Oct 12, 2023. It is now read-only.

Commit

Permalink
Mark storage version as part of pipeline (#307)
Browse files Browse the repository at this point in the history
  • Loading branch information
matthchr committed Nov 5, 2020
1 parent e0cbdd1 commit 121db21
Show file tree
Hide file tree
Showing 9 changed files with 119 additions and 98 deletions.
2 changes: 2 additions & 0 deletions hack/generator/pkg/codegen/code_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ func corePipelineStages(idFactory astmodel.IdentifierFactory, configuration *con
createStorageTypes(),
simplifyDefinitions(),

markStorageVersion(),

// Safety checks at the end:
ensureDefinitionsDoNotUseAnyTypes(),
checkForMissingStatusInformation(),
Expand Down
98 changes: 0 additions & 98 deletions hack/generator/pkg/codegen/pipeline_export_generated_code.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,6 @@ func exportPackages(outputPath string) PipelineStage {
return nil, errors.Wrapf(err, "failed to assign generated definitions to packages")
}

packages, err = MarkLatestResourceVersionsForStorage(packages)
if err != nil {
return nil, errors.Wrapf(err, "unable to mark latest resource versions for as storage versions")
}

err = writeFiles(ctx, packages, outputPath)
if err != nil {
return nil, errors.Wrapf(err, "unable to write files into %q", outputPath)
Expand Down Expand Up @@ -72,56 +67,6 @@ func CreatePackagesForDefinitions(definitions astmodel.Types) (map[astmodel.Pack
return packages, nil
}

// MarkLatestResourceVersionsForStorage marks the latest version of each resource as the storage version
func MarkLatestResourceVersionsForStorage(
packages map[astmodel.PackageReference]*astmodel.PackageDefinition) (map[astmodel.PackageReference]*astmodel.PackageDefinition, error) {

result := make(map[astmodel.PackageReference]*astmodel.PackageDefinition)

resourceLookup, err := groupResourcesByVersion(packages)
if err != nil {
return nil, err
}

for pkgRef, pkg := range packages {

resultPkg := astmodel.NewPackageDefinition(pkg.GroupName, pkg.PackageName, pkg.GeneratorVersion)
for _, def := range pkg.Definitions() {
// see if it is a resource
if resourceType, ok := def.Type().(*astmodel.ResourceType); ok {

unversionedName, err := getUnversionedName(def.Name())
if err != nil {
// should never happen as all resources have versioned names
return nil, err
}

allVersionsOfResource := resourceLookup[unversionedName]
latestVersionOfResource := allVersionsOfResource[len(allVersionsOfResource)-1]

thisPackagePath := def.Name().PackageReference.PackagePath()
latestPackagePath := latestVersionOfResource.Name().PackageReference.PackagePath()

// mark as storage version if it's the latest version
isLatestVersion := thisPackagePath == latestPackagePath
if isLatestVersion {
def = astmodel.MakeTypeDefinition(def.Name(), resourceType.MarkAsStorageVersion()).
WithDescription(def.Description())
}

resultPkg.AddDefinition(def)
} else {
// otherwise simply add it
resultPkg.AddDefinition(def)
}
}

result[pkgRef] = resultPkg
}

return result, nil
}

func writeFiles(ctx context.Context, packages map[astmodel.PackageReference]*astmodel.PackageDefinition, outputPath string) error {
var pkgs []*astmodel.PackageDefinition
for _, pkg := range packages {
Expand Down Expand Up @@ -171,49 +116,6 @@ func writeFiles(ctx context.Context, packages map[astmodel.PackageReference]*ast
return nil
}

func groupResourcesByVersion(packages map[astmodel.PackageReference]*astmodel.PackageDefinition) (map[unversionedName][]astmodel.TypeDefinition, error) {

result := make(map[unversionedName][]astmodel.TypeDefinition)

for _, pkg := range packages {
for _, def := range pkg.Definitions() {
if _, ok := def.Type().(*astmodel.ResourceType); ok {
name, err := getUnversionedName(def.Name())
if err != nil {
// this should never happen as resources will all have versioned names
return nil, errors.Wrapf(err, "Unable to extract unversioned name in groupResources")
}

result[name] = append(result[name], def)
}
}
}

// order each set of resources by package name (== by version as these are sortable dates)
for _, slice := range result {
sort.Slice(slice, func(i, j int) bool {
return slice[i].Name().PackageReference.PackageName() < slice[j].Name().PackageReference.PackageName()
})
}

return result, nil
}

func getUnversionedName(name astmodel.TypeName) (unversionedName, error) {
if localRef, ok := name.PackageReference.AsLocalPackage(); ok {
group := localRef.Group()

return unversionedName{group, name.Name()}, nil
}

return unversionedName{}, errors.New("expected local reference")
}

type unversionedName struct {
group string
name string
}

func newProgressMeter() *progressMeter {
return &progressMeter{
resetAt: time.Now(),
Expand Down
109 changes: 109 additions & 0 deletions hack/generator/pkg/codegen/pipeline_mark_storage_version.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright (c) Microsoft Corporation.
* Licensed under the MIT license.
*/

package codegen

import (
"context"
"sort"

"github.com/pkg/errors"

"github.com/Azure/k8s-infra/hack/generator/pkg/astmodel"
)

// markStorageVersion creates a PipelineStage to mark a particular version as a storage version
func markStorageVersion() PipelineStage {
return MakePipelineStage(
"markStorageVersion",
"Marking the latest version of each resource as the storage version",
func(ctx context.Context, types astmodel.Types) (astmodel.Types, error) {
updatedDefs, err := MarkLatestResourceVersionsForStorage(types)
if err != nil {
return nil, errors.Wrapf(err, "unable to mark latest resource version as storage version")
}

return updatedDefs, nil
})
}

// MarkLatestResourceVersionsForStorage marks the latest version of each resource as the storage version
func MarkLatestResourceVersionsForStorage(types astmodel.Types) (astmodel.Types, error) {

result := make(astmodel.Types)
resourceLookup, err := groupResourcesByVersion(types)
if err != nil {
return nil, err
}

for _, def := range types {
// see if it is a resource
if resourceType, ok := def.Type().(*astmodel.ResourceType); ok {

unversionedName, err := getUnversionedName(def.Name())
if err != nil {
// should never happen as all resources have versioned names
return nil, err
}

allVersionsOfResource := resourceLookup[unversionedName]
latestVersionOfResource := allVersionsOfResource[len(allVersionsOfResource)-1]

thisPackagePath := def.Name().PackageReference.PackagePath()
latestPackagePath := latestVersionOfResource.Name().PackageReference.PackagePath()

// mark as storage version if it's the latest version
isLatestVersion := thisPackagePath == latestPackagePath
if isLatestVersion {
def = astmodel.MakeTypeDefinition(def.Name(), resourceType.MarkAsStorageVersion()).
WithDescription(def.Description())
}
}
result.Add(def)
}

return result, nil
}

func groupResourcesByVersion(types astmodel.Types) (map[unversionedName][]astmodel.TypeDefinition, error) {

result := make(map[unversionedName][]astmodel.TypeDefinition)

for _, def := range types {
if astmodel.IsResourceDefinition(def) {
name, err := getUnversionedName(def.Name())
if err != nil {
// this should never happen as resources will all have versioned names
return nil, errors.Wrapf(err, "Unable to extract unversioned name in groupResources")
}

result[name] = append(result[name], def)
}
}

// order each set of resources by package name (== by version as these are sortable dates)
for _, slice := range result {
sort.Slice(slice, func(i, j int) bool {
return slice[i].Name().PackageReference.PackageName() < slice[j].Name().PackageReference.PackageName()
})
}

return result, nil
}

func getUnversionedName(name astmodel.TypeName) (unversionedName, error) {

if localRef, ok := name.PackageReference.AsLocalPackage(); ok {
group := localRef.Group()
return unversionedName{group, name.Name()}, nil
}

return unversionedName{}, errors.New("cannot get unversioned name from non-local package")
}

type unversionedName struct {
group string
name string
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/A
type A struct {
metav1.TypeMeta `json:",inline"`
Expand Down Expand Up @@ -62,6 +63,7 @@ func (aSpecArm A_SpecArm) GetType() string {
}

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/B
type B struct {
metav1.TypeMeta `json:",inline"`
Expand Down Expand Up @@ -114,6 +116,7 @@ func (bSpecArm B_SpecArm) GetType() string {
}

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/C
type C struct {
metav1.TypeMeta `json:",inline"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/FakeResource
type FakeResource struct {
metav1.TypeMeta `json:",inline"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/FakeResource
type FakeResource struct {
metav1.TypeMeta `json:",inline"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/FakeResource
type FakeResource struct {
metav1.TypeMeta `json:",inline"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/FakeResource
type FakeResource struct {
metav1.TypeMeta `json:",inline"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
)

// +kubebuilder:object:root=true
// +kubebuilder:storageversion
//Generated from: https://test.test/schemas/2020-01-01/test.json#/resourceDefinitions/FakeResource
type FakeResource struct {
metav1.TypeMeta `json:",inline"`
Expand Down

0 comments on commit 121db21

Please sign in to comment.