diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80e6524..83f5f40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,19 +4,6 @@ on: pull_request: jobs: -# golangci: -# name: Lint with golangci-lint -# runs-on: ubuntu-latest -# steps: -# - name: Checkout code -# uses: actions/checkout@v3 -# - name: Install Go -# uses: actions/setup-go@v4 -# with: -# go-version: 1.19 -# - uses: golangci/golangci-lint-action@v3 -# with: -# version: v1.49.0 vertify: name: Vertify import alias, vendor, codegen, crds runs-on: ubuntu-latest @@ -36,6 +23,7 @@ jobs: with: go-version: 1.19 - run: go mod tidy && go mod vendor + - run: hack/verify-staticcheck.sh - run: hack/verify-import-aliases.sh - run: hack/verify-vendor.sh - run: hack/verify-crds.sh diff --git a/.gitignore b/.gitignore index 73baa4a..fd063b5 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,4 @@ # Go workspace file go.work ./bin/* -./bin +bin/ diff --git a/README.md b/README.md index 2e7ad51..78631a4 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,23 @@ # runtime-copilot The main function of the runtime copilot is to assist the operation of the container runtime component (containerd), specifically for adding or deleting non-safe registries. +## Introduction + +This project is a runtime copilot, auxiliary manager runtime, current function introduce the config insecure registry to runtime(such as: containerd、docker、cri-o), It mainly has the following functions: + +[ ] Manager insecure registry in runtime. +[ ] Upgrade runtime version. +[ ] Replace runtime with another runtime. +[ ] Manager runtime plugins. + +### Manager insecure registry + +| Runtime | Support | +| --- | --- | +| containerd | Yes | +| docker | No | +| cri-o | No | + ## Usage [Helm](https://helm.sh) must be installed to use the charts. Please refer to @@ -30,7 +47,7 @@ To uninstall the chart: We add `10.6..112.191` this insecret registry to containerd, we can define yaml content follow file. ```yaml -apiVersion: config.registry.runtime.copilot.io/v1alpha1 +apiVersion: registry.runtime.x-copilot.io/v1alpha1 kind: RegistryConfigs metadata: name: registryconfigs-sample diff --git a/config.toml b/config.toml deleted file mode 100644 index 3f3e97e..0000000 --- a/config.toml +++ /dev/null @@ -1,7 +0,0 @@ -server = "https://10.6.112.191" - -[host] - - [host."https://10.6.112.191"] - Capabilities = ["pull", "push"] - skip_verify = true diff --git a/config/crd/bases/config.registry.runtime.copilot.io_noderegistryconfigs.yaml b/config/crd/bases/config.registry.runtime.copilot.io_noderegistryconfigs.yaml deleted file mode 100644 index 2d2277b..0000000 --- a/config/crd/bases/config.registry.runtime.copilot.io_noderegistryconfigs.yaml +++ /dev/null @@ -1,163 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null - name: noderegistryconfigs.config.registry.runtime.copilot.io -spec: - group: config.registry.runtime.copilot.io - names: - kind: NodeRegistryConfigs - listKind: NodeRegistryConfigsList - plural: noderegistryconfigs - singular: noderegistryconfigs - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodeRegistryConfigs is the Schema for the noderegistryconfigs - API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeRegistryConfigsSpec defines the desired state of NodeRegistryConfigs - properties: - hostConfigs: - description: HostConfigs store the per-host configuration - items: - properties: - ca_secret_ref: - type: string - capabilities: - items: - description: CapabilitieType is the type of capability - type: string - type: array - header: - additionalProperties: - type: string - type: object - override_path: - type: boolean - server: - description: Server specifies the default server. When `host` - is also specified, those hosts are tried first. - type: string - skip_verify: - type: boolean - required: - - capabilities - type: object - type: array - nodeName: - description: NodeName is registry config exec node name - type: string - retry_num: - description: set retry num - type: integer - type: - description: Type is runtime type - type: string - required: - - nodeName - - type - type: object - status: - description: NodeRegistryConfigsStatus defines the observed state of NodeRegistryConfigs - properties: - conditions: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state - of cluster Important: Run "make" to regenerate code after modifying - this file' - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - retry_num: - type: integer - state: - description: StatusState is the state of status - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/config.registry.runtime.copilot.io_registryconfigs.yaml b/config/crd/bases/config.registry.runtime.copilot.io_registryconfigs.yaml deleted file mode 100644 index 91e2806..0000000 --- a/config/crd/bases/config.registry.runtime.copilot.io_registryconfigs.yaml +++ /dev/null @@ -1,183 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.11.3 - creationTimestamp: null - name: registryconfigs.config.registry.runtime.copilot.io -spec: - group: config.registry.runtime.copilot.io - names: - kind: RegistryConfigs - listKind: RegistryConfigsList - plural: registryconfigs - singular: registryconfigs - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: RegistryConfigs is the Schema for the registryconfigs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: RegistryConfigsSpec defines the desired state of RegistryConfigs - properties: - selector: - description: Selector is used to select nodes that will be configured - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - template: - description: RegistryConfigsTemplate defines the template for the - registry config - properties: - metadata: - type: object - spec: - description: NodeHostConfigsSpec defines the host config for the - registry - properties: - hostConfigs: - items: - properties: - ca_secret_ref: - type: string - capabilities: - items: - description: CapabilitieType is the type of capability - type: string - type: array - header: - additionalProperties: - type: string - type: object - override_path: - type: boolean - server: - description: Server specifies the default server. When - `host` is also specified, those hosts are tried first. - type: string - skip_verify: - type: boolean - required: - - capabilities - type: object - type: array - retry_num: - type: integer - required: - - hostConfigs - type: object - required: - - spec - type: object - type: object - status: - description: RegistryConfigsStatus defines the observed state of RegistryConfigs - properties: - failed_nodes: - items: - properties: - num: - type: integer - runtimeType: - description: RuntimeType is the type of runtime - type: string - type: object - type: array - running_nodes: - items: - properties: - num: - type: integer - runtimeType: - description: RuntimeType is the type of runtime - type: string - type: object - type: array - state: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state - of cluster Important: Run "make" to regenerate code after modifying - this file INSERT ADDITIONAL STATUS FIELD - define observed state - of cluster Important: Run "make" to regenerate code after modifying - this file' - type: string - success_nodes: - items: - properties: - num: - type: integer - runtimeType: - description: RuntimeType is the type of runtime - type: string - type: object - type: array - total_nodes: - items: - properties: - num: - type: integer - runtimeType: - description: RuntimeType is the type of runtime - type: string - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/samples/config.registry_v1alpha1_registryconfigs.yaml b/config/samples/registry.runtime_v1alpha1_registryconfigs.yaml similarity index 91% rename from config/samples/config.registry_v1alpha1_registryconfigs.yaml rename to config/samples/registry.runtime_v1alpha1_registryconfigs.yaml index 2c06e01..6c7d711 100644 --- a/config/samples/config.registry_v1alpha1_registryconfigs.yaml +++ b/config/samples/registry.runtime_v1alpha1_registryconfigs.yaml @@ -1,4 +1,4 @@ -apiVersion: config.registry.runtime.copilot.io/v1alpha1 +apiVersion: registry.runtime.x-copilot.io/v1alpha1 kind: RegistryConfigs metadata: labels: diff --git a/config/samples/config.registry_v1alpha1_noderegistryconfigs.yaml b/config/samples/registry.runtimme_v1alpha1_noderegistryconfigs.yaml similarity index 86% rename from config/samples/config.registry_v1alpha1_noderegistryconfigs.yaml rename to config/samples/registry.runtimme_v1alpha1_noderegistryconfigs.yaml index fc454d2..dabad39 100644 --- a/config/samples/config.registry_v1alpha1_noderegistryconfigs.yaml +++ b/config/samples/registry.runtimme_v1alpha1_noderegistryconfigs.yaml @@ -1,4 +1,4 @@ -apiVersion: config.registry.runtime.copilot.io/v1alpha1 +apiVersion: registry.runtime.x-copilot.io/v1alpha1 kind: NodeRegistryConfigs metadata: labels: diff --git a/go.mod b/go.mod index 1907e69..ed26c98 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( k8s.io/client-go v0.27.1 k8s.io/code-generator v0.27.1 k8s.io/klog/v2 v2.90.1 - k8s.io/kubernetes v1.27.1 sigs.k8s.io/controller-runtime v0.15.0-alpha.0.0.20230511153903-92646a561578 sigs.k8s.io/controller-tools v0.12.0 ) diff --git a/go.sum b/go.sum index 5be86b9..af82d79 100644 --- a/go.sum +++ b/go.sum @@ -292,8 +292,6 @@ k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= -k8s.io/kubernetes v1.27.1 h1:DFeW4Lv+kh5DyYcezOzwmQAbC3VqXAxnMyZabALiRSc= -k8s.io/kubernetes v1.27.1/go.mod h1:TTwPjSCKQ+a/NTiFKRGjvOnEaQL8wIG40nsYH8Er4bA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.15.0-alpha.0.0.20230511153903-92646a561578 h1:RbJxQ21EolkZaYMSUTGnrDKqE7R6lH5ygbEGdv7toI0= diff --git a/hack/tools.go b/hack/tools.go index 944b018..fb66c13 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -1,9 +1,9 @@ -//+build tools +//go:build tools +// +build tools package hack import ( _ "k8s.io/code-generator" - _ "k8s.io/kubernetes/cmd/preferredimports" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" ) diff --git a/hack/tools/preferredimports/preferredimports.go b/hack/tools/preferredimports/preferredimports.go new file mode 100644 index 0000000..9edea2a --- /dev/null +++ b/hack/tools/preferredimports/preferredimports.go @@ -0,0 +1,281 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This code is directly lifted from the Kubernetes codebase in order to avoid relying on the k8s.io/kubernetes package. +// For reference: https://github.com/kubernetes/kubernetes/blob/release-1.22/cmd/preferredimports/preferredimports.go + +// verify that all the imports have our preferred alias(es). +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "go/ast" + "go/build" + "go/format" + "go/parser" + "go/token" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "golang.org/x/term" +) + +var ( + importAliases = flag.String("import-aliases", "hack/.import-aliases", "json file with import aliases") + confirm = flag.Bool("confirm", false, "update file with the preferred aliases for imports") + inRegex = flag.String("include-path", "(test/e2e/|test/e2e_node)", "only files with paths matching this inRegex is touched") + exRegex = flag.String("exclude-path", "(test/e2e/|test/e2e_node)", "only files with paths matching this exRegex is touched") + isTerminal = term.IsTerminal(int(os.Stdout.Fd())) + logPrefix = "" + aliases map[string]string +) + +type analyzer struct { + fset *token.FileSet // positions are relative to fset + ctx build.Context + failed bool + donePaths map[string]interface{} +} + +func newAnalyzer() *analyzer { + ctx := build.Default + ctx.CgoEnabled = true + + a := &analyzer{ + fset: token.NewFileSet(), + ctx: ctx, + donePaths: make(map[string]interface{}), + } + + return a +} + +// collect extracts test metadata from a file. +func (a *analyzer) collect(dir string) { + if _, ok := a.donePaths[dir]; ok { + return + } + a.donePaths[dir] = nil + + // Create the AST by parsing src. + fs, err := parser.ParseDir(a.fset, dir, nil, parser.AllErrors|parser.ParseComments) + if err != nil { + fmt.Fprintln(os.Stderr, "ERROR(syntax)", logPrefix, err) + a.failed = true + return + } + + for _, p := range fs { + // returns first error, but a.handleError deals with it + files := a.filterFiles(p.Files) + for _, file := range files { + replacements := make(map[string]string) + pathToFile := a.fset.File(file.Pos()).Name() + if strings.Contains(pathToFile, "generated") { + continue + } + for _, imp := range file.Imports { + importPath := strings.Replace(imp.Path.Value, "\"", "", -1) + pathSegments := strings.Split(importPath, "/") + importName := pathSegments[len(pathSegments)-1] + if imp.Name != nil { + importName = imp.Name.Name + } + if alias, ok := aliases[importPath]; ok { + if alias != importName { + if !*confirm { + fmt.Fprintf(os.Stderr, "%sERROR wrong alias for import \"%s\" should be %s in file %s\n", logPrefix, importPath, alias, pathToFile) + a.failed = true + } + replacements[importName] = alias + if imp.Name != nil { + imp.Name.Name = alias + } else { + imp.Name = ast.NewIdent(alias) + } + } + } + } + + if len(replacements) > 0 { + if *confirm { + fmt.Printf("%sReplacing imports with aliases in file %s\n", logPrefix, pathToFile) + for key, value := range replacements { + renameImportUsages(file, key, value) + } + ast.SortImports(a.fset, file) + var buffer bytes.Buffer + if err = format.Node(&buffer, a.fset, file); err != nil { + panic(fmt.Sprintf("Error formatting ast node after rewriting import.\n%s\n", err.Error())) + } + + fileInfo, err := os.Stat(pathToFile) + if err != nil { + panic(fmt.Sprintf("Error stat'ing file: %s\n%s\n", pathToFile, err.Error())) + } + + err = ioutil.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode()) + if err != nil { + panic(fmt.Sprintf("Error writing file: %s\n%s\n", pathToFile, err.Error())) + } + } + } + } + } +} + +func renameImportUsages(f *ast.File, old, new string) { + // use this to avoid renaming the package declaration, eg: + // given: package foo; import foo "bar"; foo.Baz, rename foo->qux + // yield: package foo; import qux "bar"; qux.Baz + var pkg *ast.Ident + + // Rename top-level old to new, both unresolved names + // (probably defined in another file) and names that resolve + // to a declaration we renamed. + ast.Inspect(f, func(node ast.Node) bool { + if node == nil { + return false + } + switch id := node.(type) { + case *ast.File: + pkg = id.Name + case *ast.Ident: + if pkg != nil && id == pkg { + return false + } + if id.Name == old { + id.Name = new + } + } + return true + }) +} + +func (a *analyzer) filterFiles(fs map[string]*ast.File) []*ast.File { + var files []*ast.File + for _, f := range fs { + files = append(files, f) + } + return files +} + +type collector struct { + dirs []string + inRegex *regexp.Regexp + exRegex *regexp.Regexp +} + +// handlePath walks the filesystem recursively, collecting directories, +// ignoring some unneeded directories (hidden/vendored) that are handled +// specially later. +func (c *collector) handlePath(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // Ignore hidden directories (.git, .cache, etc) + if len(path) > 1 && path[0] == '.' || + strings.Contains(path, "generated") || + // Staging code is symlinked from vendor/k8s.io, and uses import + // paths as if it were inside of vendor/. It fails typechecking + // inside of staging/, but works when typechecked as part of vendor/. + path == "staging" || + // OS-specific vendor code tends to be imported by OS-specific + // packages. We recursively typecheck imported vendored packages for + // each OS, but don't typecheck everything for every OS. + path == "vendor" || + path == "_output" || + // This is a weird one. /testdata/ is *mostly* ignored by Go, + // and this translates to kubernetes/vendor not working. + // edit/record.go doesn't compile without gopkg.in/yaml.v2 + // in $GOSRC/$GOROOT (both typecheck and the shell script). + path == "pkg/kubectl/cmd/testdata/edit" { + return filepath.SkipDir + } + if c.exRegex.MatchString(path) { + return filepath.SkipDir + } + if c.inRegex.MatchString(path) { + c.dirs = append(c.dirs, path) + } + } + return nil +} + +func main() { + flag.Parse() + args := flag.Args() + + if len(args) == 0 { + args = append(args, ".") + } + + inregex, err := regexp.Compile(*inRegex) + if err != nil { + log.Fatalf("Error compiling include regex: %v", err) + } + exregex, err := regexp.Compile(*exRegex) + if err != nil { + log.Fatalf("Error compiling exclude regex: %v", err) + } + c := collector{ + inRegex: inregex, + exRegex: exregex, + } + for _, arg := range args { + err := filepath.Walk(arg, c.handlePath) + if err != nil { + log.Fatalf("Error walking: %v", err) + } + } + sort.Strings(c.dirs) + + if len(*importAliases) > 0 { + bytes, err := ioutil.ReadFile(*importAliases) + if err != nil { + log.Fatalf("Error reading import aliases: %v", err) + } + err = json.Unmarshal(bytes, &aliases) + if err != nil { + log.Fatalf("Error loading aliases: %v", err) + } + } + if isTerminal { + logPrefix = "\r" // clear status bar when printing + } + fmt.Println("checking-imports: ") + + a := newAnalyzer() + for _, dir := range c.dirs { + if isTerminal { + fmt.Printf("\r\033[0m %-80s", dir) + } + a.collect(dir) + } + fmt.Println() + if a.failed { + os.Exit(1) + } +} diff --git a/hack/tools/tools.go b/hack/tools/tools.go new file mode 100644 index 0000000..c354eb1 --- /dev/null +++ b/hack/tools/tools.go @@ -0,0 +1,11 @@ +//go:build tools +// +build tools + +package tools + +import ( + _ "github.com/golang/mock/mockgen" + _ "github.com/onsi/ginkgo/v2" + _ "golang.org/x/tools/cmd/goimports" + _ "k8s.io/code-generator" +) diff --git a/hack/util.sh b/hack/util.sh new file mode 100755 index 0000000..c555fa0 --- /dev/null +++ b/hack/util.sh @@ -0,0 +1,461 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# This script holds common bash variables and utility functions. + +ETCD_POD_LABEL="etcd" +KUBE_CONTROLLER_POD_LABEL="kube-controller-manager" + +MIN_Go_VERSION=go1.16.0 + +# This function installs a Go tools by 'go get' command. +# Parameters: +# - $1: package name, such as "sigs.k8s.io/controller-tools/cmd/controller-gen" +# - $2: package version, such as "v0.4.1" +# Note: +# Since 'go get' command will resolve and add dependencies to current module, that may update 'go.mod' and 'go.sum' file. +# So we use a temporary directory to install the tools. +function util::install_tools() { + local package="$1" + local version="$2" + + temp_path=$(mktemp -d) + pushd "${temp_path}" >/dev/null + GO111MODULE=on go install "${package}"@"${version}" + GOPATH=$(go env GOPATH | awk -F ':' '{print $1}') + export PATH=$PATH:$GOPATH/bin + popd >/dev/null + rm -rf "${temp_path}" +} + +function util::cmd_exist { + local CMD=$(command -v ${1}) + if [[ ! -x ${CMD} ]]; then + return 1 + fi + return 0 +} + +# util::cmd_must_exist check whether command is installed. +function util::cmd_must_exist { + local CMD=$(command -v ${1}) + if [[ ! -x ${CMD} ]]; then + echo "Please install ${1} and verify they are in \$PATH." + exit 1 + fi +} + +function util::verify_go_version { + local go_version + IFS=" " read -ra go_version <<< "$(GOFLAGS='' go version)" + if [[ "${MIN_Go_VERSION}" != $(echo -e "${MIN_Go_VERSION}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then + echo "Detected go version: ${go_version[*]}." + echo "kangaroo requires ${MIN_Go_VERSION} or greater." + echo "Please install ${MIN_Go_VERSION} or later." + exit 1 + fi +} + +# util::install_environment_check will check OS and ARCH before installing +# ARCH support list: amd64,arm64 +# OS support list: linux,darwin +function util::install_environment_check { + local ARCH=${1:-} + local OS=${2:-} + if [[ "$ARCH" =~ ^(amd64|arm64)$ ]]; then + if [[ "$OS" =~ ^(linux|darwin)$ ]]; then + return 0 + fi + fi + echo "Sorry, Kpanda installation does not support $ARCH/$OS at the moment" + exit 1 +} + +# util::install_kubectl will install the given version kubectl +function util::install_kubectl { + local KUBECTL_VERSION=${1} + local ARCH=${2} + local OS=${3:-linux} + if [ -z "$KUBECTL_VERSION" ]; then + KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) + fi + echo "Installing 'kubectl ${KUBECTL_VERSION}' for you" + curl --retry 5 -sSLo ./kubectl -w "%{http_code}" https://dl.k8s.io/release/"$KUBECTL_VERSION"/bin/"$OS"/"$ARCH"/kubectl | grep '200' > /dev/null + ret=$? + if [ ${ret} -eq 0 ]; then + chmod +x ./kubectl + mkdir -p ~/.local/bin/ + mv ./kubectl ~/.local/bin/kubectl + + export PATH=$PATH:~/.local/bin + else + echo "Failed to install kubectl, can not download the binary file at https://dl.k8s.io/release/$KUBECTL_VERSION/bin/$OS/$ARCH/kubectl" + exit 1 + fi +} + +# util::install_kind will install the given version kind +function util::install_kind { + local kind_version=${1} + echo "Installing 'kind ${kind_version}' for you" + local os_name + os_name=$(go env GOOS) + local arch_name + arch_name=$(go env GOARCH) + curl --retry 5 -sSLo ./kind -w "%{http_code}" "https://qiniu-download-public.daocloud.io/Kind/${kind_version}/kind-${os_name:-linux}-${arch_name:-amd64}" | grep '200' > /dev/null + ret=$? + if [ ${ret} -eq 0 ]; then + chmod +x ./kind + mkdir -p ~/.local/bin/ + mv ./kind ~/.local/bin/kind + + export PATH=$PATH:~/.local/bin + else + echo "Failed to install kind, can not download the binary file at https://qiniu-download-public.daocloud.io/Kind/${kind_version}/kind-${os_name:-linux}-${arch_name:-amd64}" + exit 1 + fi +} + +# util::wait_for_condition blocks until the provided condition becomes true +# Arguments: +# - 1: message indicating what conditions is being waited for (e.g. 'ok') +# - 2: a string representing an eval'able condition. When eval'd it should not output +# anything to stdout or stderr. +# - 3: optional timeout in seconds. If not provided, waits forever. +# Returns: +# 1 if the condition is not met before the timeout +function util::wait_for_condition() { + local msg=$1 + # condition should be a string that can be eval'd. + local condition=$2 + local timeout=${3:-} + + local start_msg="Waiting for ${msg}" + local error_msg="[ERROR] Timeout waiting for ${msg}" + + local counter=0 + while ! eval ${condition}; do + if [[ "${counter}" = "0" ]]; then + echo -n "${start_msg}" + fi + + if [[ -z "${timeout}" || "${counter}" -lt "${timeout}" ]]; then + counter=$((counter + 1)) + if [[ -n "${timeout}" ]]; then + echo -n '.' + fi + sleep 1 + else + echo -e "\n${error_msg}" + return 1 + fi + done + + if [[ "${counter}" != "0" && -n "${timeout}" ]]; then + echo ' done' + fi +} + +# util::wait_file_exist checks if a file exists, if not, wait until timeout +function util::wait_file_exist() { + local file_path=${1} + local timeout=${2} + for ((time=0; time<${timeout}; time++)); do + if [[ -e ${file_path} ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +# util::wait_pod_ready waits for pod state becomes ready until timeout. +# Parmeters: +# - $1: pod label, such as "app=etcd" +# - $2: pod namespace, such as "kangaroo-system" +# - $3: time out, such as "200s" +function util::wait_pod_ready() { + local pod_label_key=$1 + local pod_label=$2 + local pod_namespace=$3 + local timeout=$4 + + echo "wait the $pod_label ready..." + util::wait_resource_created "pods" ${pod_label_key}=${pod_label} ${pod_namespace} + set +e + util::kubectl_with_retry wait --for=condition=Ready --timeout=${timeout} pods -l ${pod_label_key}=${pod_label} -n ${pod_namespace} + ret=$? + set -e + if [ $ret -ne 0 ];then + echo "kubectl describe info: $(kubectl describe pod -l ${pod_label_key}=${pod_label} -n ${pod_namespace})" + fi + return ${ret} +} + +function util::wait_resource_created() { + local resource_type=$1 + local label_selector=$2 + local namespace=$3 + + local resource_exists=0 + local timeout=300 + local start_time=$(date +%s) + local end_time=0 + + # Use a while loop to check if a resource exists until it times out or until it is found + while [ $resource_exists -eq 0 ]; do + # Use the kubectl get command to get the number of resources + local resource_count=$(kubectl get $resource_type -l $label_selector -n $namespace --no-headers | wc -l) + # If the number of resources is greater than zero, the resource already exists, set the flag to 1, and exit the loop + if [ $resource_count -gt 0 ]; then + echo "$resource_type with label $label_selector is created." + resource_exists=1 + break + fi + # If the number of resources is equal to zero, the resource does not exist, + # calculate the difference between the current time and the start time, and determine whether the timeout occurs + end_time=$(date +%s) + local elapsed_time=$((end_time - start_time)) + # If the timeout occurs, an error message is printed and the loop exits + # else there is no timeout, print the wait message and wait for some time before continuing the loop + if [ $elapsed_time -ge $timeout ]; then + echo "Error: timeout waiting for $resource_type with label $label_selector" + break + fi + echo "Waiting for $resource_type with label $label_selector to be created..." + sleep 5 + done +} + + +# util::kubectl_with_retry will retry if execute kubectl command failed +# tolerate kubectl command failure that may happen before the pod is created by StatefulSet/Deployment. +function util::kubectl_with_retry() { + local ret=0 + local count=0 + for i in {1..20}; do + kubectl "$@" + ret=$? + if [[ ${ret} -ne 0 ]]; then + echo "kubectl $@ failed, retrying(${i} times)" + sleep 1 + continue + else + ((count++)) + # sometimes pod status is from running to error to running + # so we need check it more times + if [[ ${count} -ge 3 ]];then + return 0 + fi + sleep 5 + continue + fi + done + + echo "kubectl $@ failed" + kubectl "$@" + return ${ret} +} + +# util::create_cluster creates a kubernetes cluster +# util::create_cluster creates a kind cluster and don't wait for control plane node to be ready. +# Parmeters: +# - $1: cluster name, such as "host" +# - $2: KUBECONFIG file, such as "/var/run/host.config" +# - $3: node docker image to use for booting the cluster, such as "kindest/node:v1.19.1" +# - $4: log file path, such as "/tmp/logs/" +function util::create_cluster() { + local cluster_name=${1} + local kubeconfig=${2} + local kind_image=${3} + local cluster_config=${4:-} + + for i in {1..20}; do + rm -f "${kubeconfig}" + util::delete_cluster "${cluster_name}" + sleep 10 + kind create cluster --name "${cluster_name}" --kubeconfig="${kubeconfig}" --image="${kind_image}" --config="${cluster_config}" || true + # Judge whether the creation of kind is completed through docker + dockername=$(docker ps | grep "${cluster_name}" || true) + if [[ -z ${dockername} ]]; then + echo "kind create cluster failed, retrying(${i} times)" + util::delete_cluster "${cluster_name}" + sleep 10 + continue + else + echo "cluster ${cluster_name} created successfully" + return 0 + fi + done +} + +# util::delete_cluster deletes kind cluster by name +# Parmeters: +# - $1: cluster name, such as "host" +function util::delete_cluster() { + local cluster_name=${1} + for i in {1..10}; do + kind delete cluster --name="${cluster_name}" || true + dockername=$(docker ps | grep "${cluster_name}" || true) + if [[ -z ${dockername} ]]; then + echo "kind delete cluster successfully" + return 0 + fi + done +} +# This function returns the IP address of a docker instance +# Parameters: +# - $1: docker instance name + +function util::get_docker_native_ipaddress(){ + local container_name=$1 + docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}" +} + +# This function returns the IP address and port of a specific docker instance's host IP +# Parameters: +# - $1: docker instance name +# Note: +# Use for getting host IP and port for cluster +# "6443/tcp" assumes that API server port is 6443 and protocol is TCP + +function util::get_docker_host_ip_port(){ + local container_name=$1 + docker inspect --format='{{range $key, $value := index .NetworkSettings.Ports "6443/tcp"}}{{if eq $key 0}}{{$value.HostIp}}:{{$value.HostPort}}{{end}}{{end}}' "${container_name}" +} + +# util::check_clusters_ready checks if a cluster is ready, if not, wait until timeout +function util::check_clusters_ready() { + local kubeconfig_path=${1} + local context_name=${2} + + echo "Waiting for kubeconfig file ${kubeconfig_path} and clusters ${context_name} to be ready..." + util::wait_file_exist "${kubeconfig_path}" 300 + util::wait_for_condition 'running' "docker inspect --format='{{.State.Status}}' ${context_name}-control-plane &> /dev/null" 300 + + kubectl config rename-context "kind-${context_name}" "${context_name}" --kubeconfig="${kubeconfig_path}" + + local os_name + os_name=$(go env GOOS) + local container_ip_port + case $os_name in + linux) container_ip_port=$(util::get_docker_native_ipaddress "${context_name}-control-plane")":6443" + ;; + darwin) container_ip_port=$(util::get_docker_host_ip_port "${context_name}-control-plane") + ;; + *) + echo "OS ${os_name} does NOT support for getting container ip in installation script" + exit 1 + esac + kubectl config set-cluster "kind-${context_name}" --server="https://${container_ip_port}" --kubeconfig="${kubeconfig_path}" + + util::wait_for_condition 'ok' "kubectl --kubeconfig ${kubeconfig_path} --context ${context_name} get --raw=/healthz &> /dev/null" 300 + util::wait_for_condition 'ok' "kubectl --kubeconfig ${kubeconfig_path} wait --for=condition=ready pod --all -n kube-system &> /dev/null" 300 +} + +# util::get_macos_ipaddress will get ip address on macos interactively, store to 'MAC_NIC_IPADDRESS' if available +MAC_NIC_IPADDRESS='' +function util::get_macos_ipaddress() { + if [[ $(go env GOOS) = "darwin" ]]; then + tmp_ip=$(ipconfig getifaddr en0 || true) + echo "" + echo " Detected that you are installing Kpanda on macOS " + echo "" + echo "It needs a Macintosh IP address to bind Kpanda API Server(port 5443)," + echo "so that member clusters can access it from docker containers, please" + echo -n "input an available IP, " + if [[ -z ${tmp_ip} ]]; then + echo "you can use the command 'ifconfig' to look for one" + tips_msg="[Enter IP address]:" + else + echo "default IP will be en0 inet addr if exists" + tips_msg="[Enter for default ${tmp_ip}]:" + fi + read -r -p "${tips_msg}" MAC_NIC_IPADDRESS + MAC_NIC_IPADDRESS=${MAC_NIC_IPADDRESS:-$tmp_ip} + if [[ "${MAC_NIC_IPADDRESS}" =~ ^(([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))\.){3}([1-9]?[0-9]|1[0-9][0-9]|2([0-4][0-9]|5[0-5]))$ ]]; then + echo "Using IP address: ${MAC_NIC_IPADDRESS}" + else + echo -e "\nError: you input an invalid IP address" + exit 1 + fi + else # non-macOS + MAC_NIC_IPADDRESS=${MAC_NIC_IPADDRESS:-} + fi +} + +function util::sync_offline_pakage() { + local PROJECT_PATH=${1} + local HELM_REPO=${2} + local REGISTRY_PASSWORD=${3} + local HELM_VERSION=${4} + local REGISTRY_IP_PORT=${5} + local BUNDLE_PATH=${6} + + # charts-syncer package + yq -i ".source.repo.url = \"${HELM_REPO}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/sync_offline_package.yaml + yq -i ".source.repo.auth.password = \"${REGISTRY_PASSWORD}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/sync_offline_package.yaml + yq -i ".target.intermediateBundlesPath = \"${BUNDLE_PATH}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/sync_offline_package.yaml + yq -i ".charts[0].versions[0] = \"${HELM_VERSION}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/sync_offline_package.yaml + + charts-syncer sync --config ${PROJECT_PATH}/test/artifacts/offline-e2e/sync_offline_package.yaml + + # start docker registry + docker run -d -p 5011:5000 --restart=always --name registry-kangaroo release-ci.daocloud.io/ghippo/registry + container_id=$(docker ps | grep -E 'kangaroo.*host-control-plane' | awk '{print $1}') + sed -i 's/xx.x.xxx.xx:xxxx/'${REGISTRY_IP_PORT}'/g' "${PROJECT_PATH}"/test/artifacts/offline-e2e/config.toml + docker cp "${PROJECT_PATH}"/test/artifacts/offline-e2e/config.toml "${container_id}":/etc/containerd/ + docker exec -i $container_id bash -c "systemctl restart containerd" + + # charts-syncer load image + yq -i ".source.intermediateBundlesPath = \"${BUNDLE_PATH}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/load-image.yaml + yq -i ".target.containerRegistry = \"${REGISTRY_IP_PORT}\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/load-image.yaml + yq -i ".target.containerRepository = \"offline.test.kangaroo/kangaroo\"" ${PROJECT_PATH}/test/artifacts/offline-e2e/load-image.yaml + cd "${PROJECT_PATH}" + charts-syncer sync --config ${PROJECT_PATH}/test/artifacts/offline-e2e/load-image.yaml + + # unpack kangaroo.bundle.tar + cd "${BUNDLE_PATH}" + tar -xvf kangaroo_${HELM_VERSION}.bundle.tar +} + +function util::cleanup_registry_kangaroo() { + registry_containerid=$(docker ps | grep "registry-kangaroo" | awk '{print $1}' || true) + if [ -n "${registry_containerid}" ] ;then + docker rm -f "${registry_containerid}" + fi +} + +function util::install_charts-sync() { + local os=$(uname -s | tr '[:upper:]' '[:lower:]') + local sync_version=0.0.5 + + if [ -z "$(which charts-syncer)" ]; then + tmp=$(mktemp -d) + pushd "${tmp}" >/dev/null + wget https://github.com/DaoCloud/charts-syncer/releases/download/v${sync_version}/charts-syncer_${sync_version}_${os}_x86_64.tar.gz + tar -xvf charts-syncer_${sync_version}_${os}_x86_64.tar.gz + sudo cp -rf charts-syncer /usr/local/bin/charts-syncer + chmod +x /usr/local/bin/charts-syncer + popd >/dev/null + rm -rf "${tmp}" + fi +} + +function util::install_yq() { + local os=$(uname -s | tr '[:upper:]' '[:lower:]') + local yq_version=v4.25.3 + + if [ -z "$(which yq)" ]; then + tmp=$(mktemp -d) + pushd "${tmp}" >/dev/null + wget https://github.com/mikefarah/yq/releases/download/${yq_version}/yq_${os}_amd64 + sudo cp -rf yq_${os}_amd64 /usr/local/bin/yq + chmod +x /usr/local/bin/yq + popd >/dev/null + rm -rf "${tmp}" + fi +} diff --git a/hack/verify-import-aliases.sh b/hack/verify-import-aliases.sh index e8f12c9..ae35d36 100755 --- a/hack/verify-import-aliases.sh +++ b/hack/verify-import-aliases.sh @@ -9,9 +9,17 @@ cd "${SCRIPT_ROOT}" ROOT_PATH=$(pwd) IMPORT_ALIASES_PATH="${ROOT_PATH}/hack/.import-aliases" +INCLUDE_PATH="(${ROOT_PATH}/cmd|${ROOT_PATH}/test|${ROOT_PATH}/pkg)" +EXCLUDE_PATH="(${ROOT_PATH}/pkg/.*/mocks)" + ret=0 -go run "k8s.io/kubernetes/cmd/preferredimports" -import-aliases "${IMPORT_ALIASES_PATH}" "${ROOT_PATH}" || ret=$? +# We can't directly install preferredimports by `go install` due to the go.mod issue: +# go install k8s.io/kubernetes/cmd/preferredimports@v1.21.3: k8s.io/kubernetes@v1.21.3 +# The go.mod file for the module providing named packages contains one or +# more replace directives. It must not contain directives that would cause +# it to be interpreted differently than if it were the main module. +go run "${ROOT_PATH}/hack/tools/preferredimports/preferredimports.go" -import-aliases "${IMPORT_ALIASES_PATH}" -include-path "${INCLUDE_PATH}" -exclude-path "${EXCLUDE_PATH}" "${ROOT_PATH}" || ret=$? if [[ $ret -ne 0 ]]; then echo "!!! Please see hack/.import-aliases for the preferred aliases for imports." >&2 exit 1 diff --git a/hack/verify-staticcheck.sh b/hack/verify-staticcheck.sh new file mode 100755 index 0000000..189078f --- /dev/null +++ b/hack/verify-staticcheck.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" +source "hack/util.sh" + +GOLANGCI_LINT_PKG="github.com/golangci/golangci-lint/cmd/golangci-lint" +GOLANGCI_LINT_VER="v1.46.2" +LINTER="golangci-lint" + +which ${LINTER} || util::install_tools ${GOLANGCI_LINT_PKG} ${GOLANGCI_LINT_VER} + +${LINTER} --version + +if ${LINTER} run --timeout=10m; then + echo 'Congratulations! All Go source files have passed staticcheck.' +else + echo # print one empty line, separate from warning messages. + echo 'Please review the above warnings.' + echo 'If the above warnings do not make sense, feel free to file an issue.' + exit 1 +fi diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go deleted file mode 100644 index a3c4518..0000000 --- a/internal/controller/suite_test.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "path/filepath" - "testing" - - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - configregistryv1alpha1 "github.com/copilot-io/runtime-copilot/api/v1alpha1" - //+kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Controller Suite") -} - -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = configregistryv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - -}) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -})