Skip to content

Commit

Permalink
CRD List Generation
Browse files Browse the repository at this point in the history
Running `make generate-crd-docs` in the root project or `make update-crdlist` in Documentation will run the crdlistgen tool. This tool rebuilds the Documentation/crdlist.rst
file by looking at the pkg/k8s/apis/cilium.io/client package's CRD List. It will also scan the rst files of Documentation to see if the CRD is being used as a header anywhere
to generate a link to it.

Signed-off-by: Daniel Hawton <daniel.hawton@solo.io>
  • Loading branch information
dhawton committed Jun 9, 2023
1 parent 89bb0a3 commit 3d5725a
Show file tree
Hide file tree
Showing 9 changed files with 254 additions and 41 deletions.
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pkg/k8s/apis/cilium.io/v2/client/crds/*.yaml linguist-generated
test/controlplane/**/v1.[0-9][0-9]/*.yaml linguist-generated
test/controlplane/services/graceful-termination/*.yaml linguist-generated
Documentation/cmdref/** linguist-generated
Documentation/crdlist.rst linguist-generated
Documentation/helm-values.rst linguist-generated
Documentation/codeowners.rst linguist-generated
Documentation/_static/* -diff
Expand Down
1 change: 1 addition & 0 deletions CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ Makefile* @cilium/build
/Documentation/conf.py @cilium/docs-structure
/Documentation/configuration/index.rst @cilium/docs-structure
/Documentation/contributing/ @cilium/contributing @cilium/docs-structure
/Documentation/crdlist.rst
/Documentation/Dockerfile @cilium/docs-structure
/Documentation/gettingstarted/demo.rst @cilium/docs-structure
/Documentation/gettingstarted/gettinghelp.rst @cilium/contributing @cilium/docs-structure
Expand Down
9 changes: 8 additions & 1 deletion Documentation/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,19 @@ update-cmdref: builder-image cilium-build ## Update the command reference docume
-$(QUIET)rm -rf cmdref/cilium*.md
$(QUIET)$(DOCKER_RUN) ./update-cmdref.sh

.PHONY: update-crdlist
update-crdlist:
@$(ECHO_GEN)crdlist
make -C ../ generate-crd-docs

codeowners.rst: $(ROOT_DIR)/CODEOWNERS
@$(ECHO_GEN)$@
$(QUIET)$(DOCKER_RUN) ./update-codeowners.sh

.PHONY: update-codeowners
update-codeowners: codeowners.rst

check: builder-image api-flaggen update-cmdref update-helm-values update-codeowners ## Validate command and Helm references, as well as policy examples.
check: builder-image api-flaggen update-cmdref update-crdlist update-helm-values update-codeowners ## Validate command and Helm references, as well as policy examples.
@$(ECHO_CHECK) cmdref
$(QUIET) ./check-cmdref.sh
@$(ECHO_CHECK) $(HELM_VALUES)
Expand All @@ -74,6 +79,8 @@ check: builder-image api-flaggen update-cmdref update-helm-values update-codeown
$(QUIET) ./check-codeowners.sh
@$(ECHO_CHECK) configuration/api-restrictions-table.rst
$(QUIET) ./check-flaggen.sh
@$(ECHO_CHECK) crdlist.rst
$(QUIET) ./check-crdlist.sh

ifeq ($(V),0)
SPHINX_OPTS += -q
Expand Down
14 changes: 14 additions & 0 deletions Documentation/check-crdlist.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash

set -o errexit
set -o nounset
set -o pipefail

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
target_file="${script_dir}/crdlist.rst"

if ! git diff --quiet -- "$target_file" ; then
git --no-pager diff -- "$target_file"
echo "HINT: to fix this, run 'make -C Documentation update-crdlist'"
exit 1
fi
17 changes: 17 additions & 0 deletions Documentation/crdlist.rst

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 1 addition & 15 deletions Documentation/internals/cilium_operator.rst
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,7 @@ CRD Registration
The default behavior of the Cilium Operator is to register the CRDs used by
Cilium. The following custom resources are registered by the Cilium Operator:

- :ref:`CiliumNetworkPolicy`
- :ref:`CiliumClusterwideNetworkPolicy`
- :ref:`CiliumEndpoint <CiliumEndpoint>`
- CiliumNode
- CiliumExternalWorkload
- CiliumIdentity
- CiliumLocalRedirectPolicy
- CiliumEgressGatewayPolicy
- CiliumEndpointSlice
- CiliumClusterwideEnvoyConfig
- CiliumEnvoyConfig
- CiliumBGPPeeringPolicy
- CiliumLoadBalancerIPPool
- CiliumNodeConfig
- CiliumCIDRGroup
.. include:: ../crdlist.rst

IPAM
~~~~
Expand Down
3 changes: 3 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -640,6 +640,9 @@ update-authors: ## Update AUTHORS file for Cilium repository.
@contrib/scripts/extract_authors.sh >> AUTHORS
@cat .authors.aux >> AUTHORS

generate-crd-docs: ## Generate CRD List for documentation
$(QUIET)$(GO) run ./tools/crdlistgen

test-docs: ## Build HTML documentation.
$(MAKE) -C Documentation html

Expand Down
110 changes: 85 additions & 25 deletions pkg/k8s/apis/cilium.io/client/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,40 +93,100 @@ var (
comparableCRDSchemaVersion = versioncheck.MustVersion(k8sconst.CustomResourceDefinitionSchemaVersion)
)

type crdCreationFn func(clientset apiextensionsclient.Interface) error
type CRDList struct {
Name string
FullName string
}

// Returns a map of CRDs
func CustomResourceDefinitionList() map[string]*CRDList {
return map[string]*CRDList{
synced.CRDResourceName(k8sconstv2.CNPName): {
Name: CNPCRDName,
FullName: k8sconstv2.CNPName,
},
synced.CRDResourceName(k8sconstv2.CCNPName): {
Name: CCNPCRDName,
FullName: k8sconstv2.CCNPName,
},
synced.CRDResourceName(k8sconstv2.CNName): {
Name: CNCRDName,
FullName: k8sconstv2.CNName,
},
synced.CRDResourceName(k8sconstv2.CIDName): {
Name: CIDCRDName,
FullName: k8sconstv2.CIDName,
},
synced.CRDResourceName(k8sconstv2.CEPName): {
Name: CEPCRDName,
FullName: k8sconstv2.CEPName,
},
synced.CRDResourceName(k8sconstv2.CEWName): {
Name: CEWCRDName,
FullName: k8sconstv2.CEWName,
},
synced.CRDResourceName(k8sconstv2.CLRPName): {
Name: CLRPCRDName,
FullName: k8sconstv2.CLRPName,
},
synced.CRDResourceName(k8sconstv2.CEGPName): {
Name: CEGPCRDName,
FullName: k8sconstv2.CEGPName,
},
synced.CRDResourceName(k8sconstv2alpha1.CESName): {
Name: CESCRDName,
FullName: k8sconstv2alpha1.CESName,
},
synced.CRDResourceName(k8sconstv2.CCECName): {
Name: CCECCRDName,
FullName: k8sconstv2.CCECName,
},
synced.CRDResourceName(k8sconstv2.CECName): {
Name: CECCRDName,
FullName: k8sconstv2.CECName,
},
synced.CRDResourceName(k8sconstv2alpha1.BGPPName): {
Name: BGPPCRDName,
FullName: k8sconstv2alpha1.BGPPName,
},
synced.CRDResourceName(k8sconstv2alpha1.LBIPPoolName): {
Name: LBIPPoolCRDName,
FullName: k8sconstv2alpha1.LBIPPoolName,
},
synced.CRDResourceName(k8sconstv2alpha1.CNCName): {
Name: CNCCRDName,
FullName: k8sconstv2alpha1.CNCName,
},
synced.CRDResourceName(k8sconstv2alpha1.CCGName): {
Name: CCGCRDName,
FullName: k8sconstv2alpha1.CCGName,
},
synced.CRDResourceName(k8sconstv2alpha1.L2AnnouncementName): {
Name: L2AnnouncementCRDName,
FullName: k8sconstv2alpha1.L2AnnouncementName,
},
synced.CRDResourceName(k8sconstv2alpha1.CPIPName): {
Name: CPIPCRDName,
FullName: k8sconstv2alpha1.CPIPName,
},
}
}

// CreateCustomResourceDefinitions creates our CRD objects in the Kubernetes
// cluster.
func CreateCustomResourceDefinitions(clientset apiextensionsclient.Interface) error {
g, _ := errgroup.WithContext(context.Background())

resourceToCreateFnMapping := map[string]crdCreationFn{
synced.CRDResourceName(k8sconstv2.CNPName): createCRD(CNPCRDName, k8sconstv2.CNPName),
synced.CRDResourceName(k8sconstv2.CCNPName): createCRD(CCNPCRDName, k8sconstv2.CCNPName),
synced.CRDResourceName(k8sconstv2.CNName): createCRD(CNCRDName, k8sconstv2.CNName),
synced.CRDResourceName(k8sconstv2.CIDName): createCRD(CIDCRDName, k8sconstv2.CIDName),
synced.CRDResourceName(k8sconstv2.CEPName): createCRD(CEPCRDName, k8sconstv2.CEPName),
synced.CRDResourceName(k8sconstv2.CEWName): createCRD(CEWCRDName, k8sconstv2.CEWName),
synced.CRDResourceName(k8sconstv2.CLRPName): createCRD(CLRPCRDName, k8sconstv2.CLRPName),
synced.CRDResourceName(k8sconstv2.CEGPName): createCRD(CEGPCRDName, k8sconstv2.CEGPName),
synced.CRDResourceName(k8sconstv2alpha1.CESName): createCRD(CESCRDName, k8sconstv2alpha1.CESName),
synced.CRDResourceName(k8sconstv2.CCECName): createCRD(CCECCRDName, k8sconstv2.CCECName),
synced.CRDResourceName(k8sconstv2.CECName): createCRD(CECCRDName, k8sconstv2.CECName),
synced.CRDResourceName(k8sconstv2alpha1.BGPPName): createCRD(BGPPCRDName, k8sconstv2alpha1.BGPPName),
synced.CRDResourceName(k8sconstv2alpha1.LBIPPoolName): createCRD(LBIPPoolCRDName, k8sconstv2alpha1.LBIPPoolName),
synced.CRDResourceName(k8sconstv2alpha1.CNCName): createCRD(CNCCRDName, k8sconstv2alpha1.CNCName),
synced.CRDResourceName(k8sconstv2alpha1.CCGName): createCRD(CCGCRDName, k8sconstv2alpha1.CCGName),
synced.CRDResourceName(k8sconstv2alpha1.L2AnnouncementName): createCRD(L2AnnouncementCRDName, k8sconstv2alpha1.L2AnnouncementName),
synced.CRDResourceName(k8sconstv2alpha1.CPIPName): createCRD(CPIPCRDName, k8sconstv2alpha1.CPIPName),
}
crds := CustomResourceDefinitionList()

for _, r := range synced.AllCiliumCRDResourceNames() {
fn, ok := resourceToCreateFnMapping[r]
if !ok {
if crd, ok := crds[r]; ok {
g.Go(func() error {
return createCRD(crd.Name, crd.FullName)(clientset)
})
} else {
log.Fatalf("Unknown resource %s. Please update pkg/k8s/apis/cilium.io/client to understand this type.", r)
}
g.Go(func() error {
return fn(clientset)
})
}

return g.Wait()
Expand Down
124 changes: 124 additions & 0 deletions tools/crdlistgen/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package main

import (
"bufio"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"

operatorServer "github.com/cilium/cilium/api/v1/operator/server"
"github.com/cilium/cilium/pkg/hive"
"github.com/cilium/cilium/pkg/hive/cell"
"github.com/cilium/cilium/pkg/k8s/apis/cilium.io/client"
)

var (
MainCell = cell.Module(
"main",
"Main module for generating CRD Lists",
cell.Invoke(printCRDList),
)

Hive = hive.New(
operatorServer.SpecCell,
MainCell,
)
)

var ErrorBreakEarly = fmt.Errorf("break early")

func printCRDList(
opSpec *operatorServer.Spec,
shutdown hive.Shutdowner,
) error {
list := client.CustomResourceDefinitionList()

crdlist := []string{}

for _, crd := range list {
crdlist = append(crdlist, cleanupCRDName(crd.Name))
}

// Sort the list
sort.Strings(crdlist)

for idx, name := range crdlist {
// We need to walk ../../Documentation rst files to look and see if the CRD name is a header in the format of `.. _ <name>:`, if so
// add `ref:` to the name so it will link to the CRD in the docs.
err := filepath.WalkDir("./Documentation", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

if filepath.Ext(path) == ".rst" {
match, err := grepFile(path, ".. _"+name+":")
if err != nil {
return err
}
// We can stop walking the documentation as we already know there is a match, so we send an ErrorBreakEarly and ignore that on the other side
// as WalkDir will keep running until it returns an error or there are no more files to walk.
if match {
// Change the name to a ref also specifically override the link text, as a couple headers add " CRD" to the text which causes the link to not be uniform.
crdlist[idx] = ":ref:`" + name + "<" + name + ">`"
return ErrorBreakEarly
}
}

return nil
})
if err != nil && !errors.Is(err, ErrorBreakEarly) {
return err
}
}

f, err := os.Create("Documentation/crdlist.rst")
if err != nil {
return err
}
defer f.Close()

for _, name := range crdlist {
_, err := f.WriteString(fmt.Sprintf("- %s\n", name))
if err != nil {
return err
}
}

shutdown.Shutdown()
return nil
}

// Scan file for string
func grepFile(path, search string) (bool, error) {
//fmt.Printf("searching %s for %s\n", path, search)
f, err := os.Open(path)
if err != nil {
return false, err
}
defer f.Close()

scanner := bufio.NewScanner(f)
for scanner.Scan() {
if strings.Contains(scanner.Text(), search) {
return true, nil
}
}

return false, scanner.Err()
}

// Remove the /(version) portion from the CRD Name
func cleanupCRDName(name string) string {
return strings.Split(name, "/")[0]
}

func main() {
Hive.Run()
}

0 comments on commit 3d5725a

Please sign in to comment.