From c1111d0a2cc9102ce0f1b706df8c280e6328cb31 Mon Sep 17 00:00:00 2001 From: Duan Jiong Date: Tue, 10 Nov 2020 20:31:33 +0800 Subject: [PATCH] refactor porter 1. Eip Address Management via CRD 2. refactor Porter High Availability 3. Changes to the BgpConf/BgpPeer API to be compatible with the gobgp API and to support viewing status. 4. Some other changes Signed-off-by: Duan Jiong --- Makefile | 77 +- README.md | 2 +- api/v1alpha1/eip_types.go | 5 - api/v1alpha2/bgpconf_types.go | 117 ++ api/v1alpha2/bgppeer_types.go | 253 ++++ api/v1alpha2/eip_types.go | 214 ++++ api/v1alpha2/groupversion_info.go | 35 + api/v1alpha2/types_test.go | 130 ++ api/v1alpha2/zz_generated.deepcopy.go | 707 +++++++++++ cmd/agent/Dockerfile | 2 +- cmd/agent/main.go | 50 +- cmd/manager/app/manager.go | 101 +- cmd/manager/app/options/generic.go | 26 - cmd/manager/app/options/options.go | 15 +- .../bases/network.kubesphere.io_bgpconfs.yaml | 187 ++- .../bases/network.kubesphere.io_bgppeers.yaml | 467 ++++++- .../crd/bases/network.kubesphere.io_eips.yaml | 188 ++- config/crd/kustomization.yaml | 8 +- config/crd/kustomizeconfig.yaml | 22 +- config/default/kustomization.yaml | 15 +- config/{release => default}/ns.yaml | 0 config/dev/agent_image_patch.yaml | 9 +- config/dev/cert.yaml | 114 ++ config/dev/kustomization.yaml | 11 +- config/dev/manager_image_patch.yaml | 12 +- config/dev/nfs_patch.yaml | 17 + config/dev/ns.yaml | 4 - config/rbac/kustomization.yaml | 8 +- config/rbac/leader_election_role.yaml | 52 +- config/rbac/leader_election_role_binding.yaml | 5 +- config/rbac/role.yaml | 17 +- config/rbac/role_binding.yaml | 7 +- config/release/agent_image_patch.yaml | 7 +- config/release/kustomization.yaml | 9 +- config/release/manager_image_patch.yaml | 14 +- config/samples/network_v1alpha1_bgpconf.yaml | 10 - config/samples/network_v1alpha1_bgppeer.yaml | 11 - config/samples/network_v1alpha1_eip.yaml | 7 - config/samples/network_v1alpha2_bgpconf.yaml | 13 + config/samples/network_v1alpha2_bgppeer.yaml | 18 + config/samples/network_v1alpha2_eip.yaml | 6 + .../samples/network_v1alpha2_eip_layer2.yaml | 9 + config/samples/workload.yaml | 53 + config/webhook/kustomization.yaml | 4 + config/webhook/manifests.yaml | 26 + config/webhook/service.yaml | 13 + config/webhook/webhook.yaml | 27 + config/workloads/agent.yaml | 19 +- config/workloads/kustomization.yaml | 2 +- config/workloads/manager.yaml | 110 +- controllers/bgp/bgpconf_controller.go | 115 -- controllers/bgp/bgppeer_controller.go | 115 -- controllers/eip/eip_controller.go | 132 -- controllers/eip/netutil.go | 63 - controllers/lb/controller.go | 199 --- controllers/lb/lb.go | 157 --- controllers/suite_test.go | 84 -- deploy/porter.yaml | 1115 ++++++++++++----- doc/bgp_config.md | 124 +- doc/compared_with_metallb.md | 99 +- doc/deploy.md | 48 + doc/deploy_baremetal.md | 143 --- doc/eip_config.md | 81 +- doc/index.md | 18 + doc/layer2.md | 137 +- doc/porter-chart.md | 298 ----- doc/router_config.md | 3 - doc/simulate_with_bird.md | 349 +++--- doc/usage.md | 184 +++ doc/usecases.md | 188 --- doc/zh/bgp_config.md | 119 +- doc/zh/compared_with_metallb.md | 87 +- doc/zh/deploy.md | 47 + doc/zh/deploy_baremetal.md | 138 -- doc/zh/eip_config.md | 87 ++ doc/zh/index.md | 18 + doc/zh/layer2.md | 131 +- doc/zh/porter-chart.md | 298 ----- doc/zh/router_config.md | 5 - doc/zh/simulate_with_bird.md | 342 ++--- doc/zh/usage.md | 176 +++ go.mod | 8 +- go.sum | 96 ++ hack/deploy.sh | 23 - hack/install_tools.sh | 14 +- hack/test.sh | 56 - pkg/bgp/serverd/bgp_test.go | 119 -- pkg/bgp/serverd/conf.go | 52 - pkg/bgp/serverd/options.go | 32 - pkg/bgp/serverd/peers.go | 184 --- pkg/bgp/serverd/serverd.go | 249 ---- pkg/bgp/serverd/serverd_suite_test.go | 13 - pkg/constant/constants.go | 27 +- pkg/controllers/bgp/bgpconf_controller.go | 176 +++ pkg/controllers/bgp/bgppeer_controller.go | 187 +++ pkg/controllers/ipam/ipam.go | 392 ++++++ pkg/controllers/ipam/ipam_test.go | 291 +++++ pkg/controllers/lb/controller.go | 389 ++++++ pkg/controllers/lb/suite_test.go | 533 ++++++++ pkg/errors/errors.go | 35 - pkg/ipam/datastore.go | 344 ----- pkg/ipam/eip_status.go | 67 - pkg/ipam/ipam.go | 115 -- pkg/ipam/ipam_suite_test.go | 13 - pkg/ipam/ipam_test.go | 108 -- pkg/ipam/protocol.go | 55 - pkg/kubeutil/kubeutil.go | 49 - pkg/layer2/arp.go | 192 --- pkg/layer2/arp_test.go | 29 - pkg/layer2/doc.go | 3 - pkg/layer2/layer2_suite_test.go | 13 - pkg/layer2/responder.go | 55 - pkg/leader-elector/election.go | 62 + pkg/leader-elector/options.go | 1 + pkg/log/log.go | 25 +- pkg/log/options.go | 28 + pkg/manager/client/client.go | 7 + pkg/manager/generic.go | 55 + pkg/nettool/iptables.go | 24 - pkg/speaker/bgp/bgp_test.go | 191 +++ pkg/speaker/bgp/conf.go | 23 + pkg/speaker/bgp/gobgpd.go | 85 ++ pkg/speaker/bgp/options.go | 26 + pkg/speaker/bgp/path.go | 215 ++++ pkg/speaker/bgp/peers.go | 123 ++ pkg/speaker/layer2/arp.go | 267 ++++ pkg/speaker/layer2/arp_test.go | 140 +++ pkg/speaker/layer2/doc.go | 1 + pkg/speaker/layer2/responder.go | 63 + pkg/speaker/speaker.go | 127 ++ pkg/test/ip_test.go | 20 - pkg/test/porter_suite_test.go | 13 - pkg/test/util.go | 51 + pkg/util/cidr/cidr.go | 242 ---- pkg/util/cidr/cidr_suite_test.go | 13 - pkg/util/cidr/cidr_test.go | 334 ----- pkg/util/ip.go | 166 --- pkg/util/ip_test.go | 44 - pkg/util/string.go | 20 - pkg/util/util.go | 53 + pkg/util/util_suite_test.go | 4 + pkg/util/{string_test.go => util_test.go} | 0 pkg/validate/validate.go | 48 +- 143 files changed, 8703 insertions(+), 6022 deletions(-) create mode 100644 api/v1alpha2/bgpconf_types.go create mode 100644 api/v1alpha2/bgppeer_types.go create mode 100644 api/v1alpha2/eip_types.go create mode 100644 api/v1alpha2/groupversion_info.go create mode 100644 api/v1alpha2/types_test.go create mode 100644 api/v1alpha2/zz_generated.deepcopy.go delete mode 100644 cmd/manager/app/options/generic.go rename config/{release => default}/ns.yaml (100%) create mode 100644 config/dev/cert.yaml create mode 100644 config/dev/nfs_patch.yaml delete mode 100644 config/dev/ns.yaml delete mode 100644 config/samples/network_v1alpha1_bgpconf.yaml delete mode 100644 config/samples/network_v1alpha1_bgppeer.yaml delete mode 100644 config/samples/network_v1alpha1_eip.yaml create mode 100644 config/samples/network_v1alpha2_bgpconf.yaml create mode 100644 config/samples/network_v1alpha2_bgppeer.yaml create mode 100644 config/samples/network_v1alpha2_eip.yaml create mode 100644 config/samples/network_v1alpha2_eip_layer2.yaml create mode 100644 config/samples/workload.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/service.yaml create mode 100644 config/webhook/webhook.yaml delete mode 100644 controllers/bgp/bgpconf_controller.go delete mode 100644 controllers/bgp/bgppeer_controller.go delete mode 100644 controllers/eip/eip_controller.go delete mode 100644 controllers/eip/netutil.go delete mode 100644 controllers/lb/controller.go delete mode 100644 controllers/lb/lb.go delete mode 100644 controllers/suite_test.go create mode 100644 doc/deploy.md delete mode 100644 doc/deploy_baremetal.md create mode 100644 doc/index.md delete mode 100644 doc/porter-chart.md delete mode 100644 doc/router_config.md create mode 100644 doc/usage.md delete mode 100644 doc/usecases.md create mode 100644 doc/zh/deploy.md delete mode 100644 doc/zh/deploy_baremetal.md create mode 100644 doc/zh/eip_config.md create mode 100644 doc/zh/index.md delete mode 100644 doc/zh/porter-chart.md delete mode 100644 doc/zh/router_config.md create mode 100644 doc/zh/usage.md delete mode 100755 hack/deploy.sh delete mode 100755 hack/test.sh delete mode 100644 pkg/bgp/serverd/bgp_test.go delete mode 100644 pkg/bgp/serverd/conf.go delete mode 100644 pkg/bgp/serverd/options.go delete mode 100644 pkg/bgp/serverd/peers.go delete mode 100644 pkg/bgp/serverd/serverd.go delete mode 100644 pkg/bgp/serverd/serverd_suite_test.go create mode 100644 pkg/controllers/bgp/bgpconf_controller.go create mode 100644 pkg/controllers/bgp/bgppeer_controller.go create mode 100644 pkg/controllers/ipam/ipam.go create mode 100644 pkg/controllers/ipam/ipam_test.go create mode 100644 pkg/controllers/lb/controller.go create mode 100644 pkg/controllers/lb/suite_test.go delete mode 100644 pkg/errors/errors.go delete mode 100644 pkg/ipam/datastore.go delete mode 100644 pkg/ipam/eip_status.go delete mode 100644 pkg/ipam/ipam.go delete mode 100644 pkg/ipam/ipam_suite_test.go delete mode 100644 pkg/ipam/ipam_test.go delete mode 100644 pkg/ipam/protocol.go delete mode 100644 pkg/kubeutil/kubeutil.go delete mode 100644 pkg/layer2/arp.go delete mode 100644 pkg/layer2/arp_test.go delete mode 100644 pkg/layer2/doc.go delete mode 100644 pkg/layer2/layer2_suite_test.go delete mode 100644 pkg/layer2/responder.go create mode 100644 pkg/leader-elector/election.go create mode 100644 pkg/leader-elector/options.go create mode 100644 pkg/log/options.go create mode 100644 pkg/manager/client/client.go create mode 100644 pkg/manager/generic.go create mode 100644 pkg/speaker/bgp/bgp_test.go create mode 100644 pkg/speaker/bgp/conf.go create mode 100644 pkg/speaker/bgp/gobgpd.go create mode 100644 pkg/speaker/bgp/options.go create mode 100644 pkg/speaker/bgp/path.go create mode 100644 pkg/speaker/bgp/peers.go create mode 100644 pkg/speaker/layer2/arp.go create mode 100644 pkg/speaker/layer2/arp_test.go create mode 100644 pkg/speaker/layer2/doc.go create mode 100644 pkg/speaker/layer2/responder.go create mode 100644 pkg/speaker/speaker.go delete mode 100644 pkg/test/ip_test.go delete mode 100644 pkg/test/porter_suite_test.go create mode 100644 pkg/test/util.go delete mode 100644 pkg/util/cidr/cidr.go delete mode 100644 pkg/util/cidr/cidr_suite_test.go delete mode 100644 pkg/util/cidr/cidr_test.go delete mode 100644 pkg/util/ip.go delete mode 100644 pkg/util/ip_test.go delete mode 100644 pkg/util/string.go create mode 100644 pkg/util/util.go rename pkg/util/{string_test.go => util_test.go} (100%) diff --git a/Makefile b/Makefile index 67f4222ce..4c69a1059 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # Image URL to use all building/pushing image targets -IMG_MANAGER ?= kubespheredev/porter:v0.3-dev -IMG_AGENT ?= kubespheredev/porter-agent:v0.3-dev -NAMESPACE ?= porter-system +IMG_MANAGER ?= kubespheredev/porter:v0.4 +IMG_AGENT ?= kubespheredev/porter-agent:v0.4 +BRANCH ?= release CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -15,42 +15,43 @@ endif all: manager +# Run go fmt against code +fmt: + go fmt ./pkg/... ./cmd/... ./api/... ./pkg/controllers/... + +# Run go vet against code +vet: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go vet ./pkg/... ./cmd/... ./pkg/controllers/... + # Run tests test: fmt vet - go test -v ./api/... ./controllers/... ./pkg/... -coverprofile cover.out + go test -v ./api/... ./pkg/controllers/... ./pkg/... -coverprofile cover.out # Build manager binary manager: fmt vet - go build -o bin/manager github.com/kubesphere/porter/cmd/manager + #CGO_ENABLED=0 go build -a -ldflags '-extldflags "-static"' -o bin/manager github.com/kubesphere/porter/cmd/manager + CGO_ENABLED=0 go build -o bin/manager github.com/kubesphere/porter/cmd/manager -# Install CRDs into a cluster -install: manifests - kubectl apply -f config/crd/bases -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests - kubectl apply -f config/crd/bases - kustomize build config/default | kubectl apply -f - +deploy: generate +ifeq ($(uname), Darwin) + sed -i '' -e 's@image: .*@image: '"${IMG_AGENT}"'@' ./config/${BRANCH}/agent_image_patch.yaml + sed -i '' -e 's@image: .*@image: '"${IMG_MANAGER}"'@' ./config/${BRANCH}/manager_image_patch.yaml +else + sed -i -e 's@image: .*@image: '"${IMG_AGENT}"'@' ./config/${BRANCH}/agent_image_patch.yaml + sed -i -e 's@image: .*@image: '"${IMG_MANAGER}"'@' ./config/${BRANCH}/manager_image_patch.yaml +endif + kustomize build config/${BRANCH} -o deploy/porter.yaml + @echo "Done, the yaml is in deploy folder named 'porter.yaml'" # Generate code generate: controller-gen $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths=./api/... - -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./api/..." paths="./controllers/..." output:crd:artifacts:config=config/crd/bases -# Run go fmt against code -fmt: - go fmt ./pkg/... ./cmd/... ./api/... ./controllers/... - -# Run go vet against code -vet: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go vet ./pkg/... ./cmd/... ./controllers/... - + $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=porter-manager-role webhook paths="./api/..." paths="./pkg/controllers/..." output:crd:artifacts:config=config/crd/bases controller-gen: ifeq (, $(shell which controller-gen)) - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.0 + go get sigs.k8s.io/controller-tools/cmd/controller-gen@ CONTROLLER_GEN=$(GOBIN)/controller-gen else CONTROLLER_GEN=$(shell which controller-gen) @@ -59,25 +60,15 @@ endif clean-up: ./hack/cleanup.sh -release: - export DOCKER_CLI_EXPERIMENTAL=enabled - GOOS=linux GOARCH=amd64 go build -o bin/manager-linux-amd64 github.com/kubesphere/porter/cmd/manager - GOOS=linux GOARCH=amd64 go build -o bin/agent-linux-amd64 github.com/kubesphere/porter/cmd/agent - GOOS=linux GOARCH=arm64 go build -o bin/manager-linux-arm64 github.com/kubesphere/porter/cmd/manager - GOOS=linux GOARCH=arm64 go build -o bin/agent-linux-arm64 github.com/kubesphere/porter/cmd/agent - docker buildx build --platform linux/amd64,linux/arm64 -t ${IMG_AGENT} -f ./cmd/agent/Dockerfile . --push - docker buildx build --platform linux/amd64,linux/arm64 -t ${IMG_MANAGER} -f ./cmd/manager/Dockerfile . --push - -ifeq ($(uname), Darwin) - sed -i '' -e 's@image: .*@image: '"${IMG_AGENT}"'@' ./config/release/agent_image_patch.yaml - sed -i '' -e 's@image: .*@image: '"${IMG_MANAGER}"'@' ./config/release/manager_image_patch.yaml -else - sed -i -e 's@image: .*@image: '"${IMG_AGENT}"'@' ./config/release/agent_image_patch.yaml - sed -i -e 's@image: .*@image: '"${IMG_MANAGER}"'@' ./config/release/manager_image_patch.yaml -endif - kustomize build config/release -o deploy/porter.yaml - @echo "Done, the yaml is in deploy folder named 'porter.yaml'" +release: deploy + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/manager-linux-amd64 github.com/kubesphere/porter/cmd/manager + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/agent-linux-amd64 github.com/kubesphere/porter/cmd/agent + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/manager-linux-arm64 github.com/kubesphere/porter/cmd/manager + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o bin/agent-linux-arm64 github.com/kubesphere/porter/cmd/agent + DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform linux/amd64,linux/arm64 -t ${IMG_AGENT} -f ./cmd/agent/Dockerfile . --push + DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --platform linux/amd64,linux/arm64 -t ${IMG_MANAGER} -f ./cmd/manager/Dockerfile . --push install-travis: + echo "install kubebuilder/kustomize etc." chmod +x ./hack/*.sh ./hack/install_tools.sh diff --git a/README.md b/README.md index ff5f9ccf0..7be72c409 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ However, the service is hard to expose in a bare metal cluster since Kubernetes - ECMP routing load balancing - BGP dynamic routing configuration -- VIP management +- EIP management - LoadBalancerIP assignment in Kubernetes services - Installation with Helm Chart - Dynamic BGP server configuration through CRD diff --git a/api/v1alpha1/eip_types.go b/api/v1alpha1/eip_types.go index 06c56ed17..2ac6f84e5 100644 --- a/api/v1alpha1/eip_types.go +++ b/api/v1alpha1/eip_types.go @@ -16,7 +16,6 @@ limitations under the License. package v1alpha1 import ( - "github.com/kubesphere/porter/pkg/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -66,7 +65,3 @@ type EipList struct { func init() { SchemeBuilder.Register(&Eip{}, &EipList{}) } - -func (e Eip) GetEIPSize() int { - return util.GetValidAddressCount(e.Spec.Address) -} diff --git a/api/v1alpha2/bgpconf_types.go b/api/v1alpha2/bgpconf_types.go new file mode 100644 index 000000000..00faf157b --- /dev/null +++ b/api/v1alpha2/bgpconf_types.go @@ -0,0 +1,117 @@ +/* +Copyright 2019 The Kubesphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "bytes" + "encoding/json" + "github.com/golang/protobuf/jsonpb" + api "github.com/osrg/gobgp/api" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type NodeConfStatus struct { + RouterId string `json:"routerId,omitempty"` +} + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// BgpConfStatus defines the observed state of BgpConf +type BgpConfStatus struct { + NodesConfStatus map[string]NodeConfStatus `json:"nodesConfStatus,omitempty"` +} + +// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgpconfs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgpconfs/status,verbs=get;update;patch + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope=Cluster + +// BgpConf is the Schema for the bgpconfs API +type BgpConf struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BgpConfSpec `json:"spec,omitempty"` + Status BgpConfStatus `json:"status,omitempty"` +} + +func (c BgpConfSpec) ConverToGoBgpGlabalConf() (*api.Global, error) { + jsonBytes, err := json.Marshal(c) + if err != nil { + return nil, err + } + + var result api.Global + m := jsonpb.Unmarshaler{} + return &result, m.Unmarshal(bytes.NewReader(jsonBytes), &result) +} + +func ConverFromGoBgpGlobalConf(global *api.Global) (*BgpConfSpec, error) { + m := jsonpb.Marshaler{} + jsonStr, err := m.MarshalToString(global) + if err != nil { + return nil, err + } + + var result BgpConfSpec + + return &result, json.Unmarshal([]byte(jsonStr), &result) +} + +// Configuration parameters relating to the global BGP router. +type BgpConfSpec struct { + As uint32 `json:"as,omitempty"` + RouterId string `json:"routerId,omitempty"` + ListenPort int32 `json:"listenPort,omitempty"` + ListenAddresses []string `json:"listenAddresses,omitempty"` + Families []uint32 `json:"families,omitempty"` + UseMultiplePaths bool `json:"useMultiplePaths,omitempty"` + GracefulRestart *GracefulRestart `json:"gracefulRestart,omitempty"` +} + +type GracefulRestart struct { + Enabled bool `json:"enabled,omitempty"` + RestartTime uint32 `json:"restartTime,omitempty"` + HelperOnly bool `json:"helperOnly,omitempty"` + DeferralTime uint32 `json:"deferralTime,omitempty"` + NotificationEnabled bool `json:"notificationEnabled,omitempty"` + LonglivedEnabled bool `json:"longlivedEnabled,omitempty"` + StaleRoutesTime uint32 `json:"staleRoutesTime,omitempty"` + PeerRestartTime uint32 `json:"peerRestartTime,omitempty"` + PeerRestarting bool `json:"peerRestarting,omitempty"` + LocalRestarting bool `json:"localRestarting,omitempty"` + Mode string `json:"mode,omitempty"` +} + +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// BgpConfList contains a list of BgpConf +type BgpConfList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BgpConf `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BgpConf{}, &BgpConfList{}) +} diff --git a/api/v1alpha2/bgppeer_types.go b/api/v1alpha2/bgppeer_types.go new file mode 100644 index 000000000..c18324a9d --- /dev/null +++ b/api/v1alpha2/bgppeer_types.go @@ -0,0 +1,253 @@ +/* +Copyright 2019 The Kubesphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "bytes" + "encoding/json" + "github.com/golang/protobuf/jsonpb" + api "github.com/osrg/gobgp/api" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +type Message struct { + Notification string `json:"notification,omitempty"` + Update string `json:"update,omitempty"` + Open string `json:"open,omitempty"` + Keepalive string `json:"keepalive,omitempty"` + Refresh string `json:"refresh,omitempty"` + Discarded string `json:"discarded,omitempty"` + Total string `json:"total,omitempty"` + WithdrawUpdate string `json:"withdrawUpdate,omitempty"` + WithdrawPrefix string `json:"withdrawPrefix,omitempty"` +} + +type Messages struct { + Received *Message `json:"received,omitempty"` + Sent *Message `json:"sent,omitempty"` +} + +type Queues struct { + Input uint32 `json:"input,omitempty"` + Output uint32 `json:"output,omitempty"` +} + +type PeerState struct { + AuthPassword string `json:"authPassword,omitempty"` + Description string `json:"description,omitempty"` + LocalAs uint32 `json:"localAs,omitempty"` + Messages *Messages `json:"messages,omitempty"` + NeighborAddress string `json:"neighborAddress,omitempty"` + PeerAs uint32 `json:"peerAs,omitempty"` + PeerGroup string `json:"peerGroup,omitempty"` + PeerType uint32 `json:"peerType,omitempty"` + Queues *Queues `json:"queues,omitempty"` + RemovePrivateAs uint32 `json:"removePrivateAs,omitempty"` + RouteFlapDamping bool `json:"routeFlapDamping,omitempty"` + SendCommunity uint32 `json:"sendCommunity,omitempty"` + SessionState string `json:"sessionState,omitempty"` + AdminState string `json:"adminState,omitempty"` + OutQ uint32 `json:"outQ,omitempty"` + Flops uint32 `json:"flops,omitempty"` + RouterId string `json:"routerId,omitempty"` +} + +type TimersState struct { + ConnectRetry string `json:"connectRetry,omitempty"` + HoldTime string `json:"holdTime,omitempty"` + KeepaliveInterval string `json:"keepaliveInterval,omitempty"` + MinimumAdvertisementInterval string `json:"minimumAdvertisementInterval,omitempty"` + NegotiatedHoldTime string `json:"negotiatedHoldTime,omitempty"` + Uptime string `json:"uptime,omitempty"` + Downtime string `json:"downtime,omitempty"` +} + +type NodePeerStatus struct { + PeerState PeerState `json:"peerState,omitempty"` + TimersState TimersState `json:"timersState,omitempty"` +} + +// BgpPeerStatus defines the observed state of BgpPeer +type BgpPeerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + NodesPeerStatus map[string]NodePeerStatus `json:"nodesPeerStatus,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:scope=Cluster + +// BgpPeer is the Schema for the bgppeers API +type BgpPeer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BgpPeerSpec `json:"spec,omitempty"` + Status BgpPeerStatus `json:"status,omitempty"` +} + +type Timers struct { + Config *TimersConfig `json:"config,omitempty"` +} + +// https://stackoverflow.com/questions/21151765/cannot-unmarshal-string-into-go-value-of-type-int64 +type TimersConfig struct { + ConnectRetry string `json:"connectRetry,omitempty"` + HoldTime string `json:"holdTime,omitempty"` + KeepaliveInterval string `json:"keepaliveInterval,omitempty"` + MinimumAdvertisementInterval string `json:"minimumAdvertisementInterval,omitempty"` +} + +type PeerConf struct { + AuthPassword string `json:"authPassword,omitempty"` + Description string `json:"description,omitempty"` + LocalAs uint32 `json:"localAs,omitempty"` + NeighborAddress string `json:"neighborAddress,omitempty"` + PeerAs uint32 `json:"peerAs,omitempty"` + PeerGroup string `json:"peerGroup,omitempty"` + PeerType uint32 `json:"peerType,omitempty"` + RemovePrivateAs string `json:"removePrivateAs,omitempty"` + RouteFlapDamping bool `json:"routeFlapDamping,omitempty"` + SendCommunity uint32 `json:"sendCommunity,omitempty"` + NeighborInterface string `json:"neighborInterface,omitempty"` + Vrf string `json:"vrf,omitempty"` + AllowOwnAs uint32 `json:"allowOwnAs,omitempty"` + ReplacePeerAs bool `json:"replacePeerAs,omitempty"` + AdminDown bool `json:"adminDown,omitempty"` +} + +type Transport struct { + MtuDiscovery bool `json:"mtuDiscovery,omitempty"` + PassiveMode bool `json:"passiveMode,omitempty"` + RemoteAddress string `json:"remoteAddress,omitempty"` + RemotePort uint32 `json:"remotePort,omitempty"` + TcpMss uint32 `json:"tcpMss,omitempty"` +} + +type MpGracefulRestartConfig struct { + Enabled bool `json:"enabled,omitempty"` +} + +type MpGracefulRestart struct { + Config *MpGracefulRestartConfig `json:"config,omitempty"` +} + +type Family struct { + Afi string `json:"afi,omitempty"` + Safi string `json:"safi,omitempty"` +} + +type AfiSafiConfig struct { + Family *Family `json:"family,omitempty"` + Enabled bool `json:"enabled,omitempty"` +} + +type AddPathsConfig struct { + Receive bool `json:"receive,omitempty"` + SendMax uint32 `json:"sendMax,omitempty"` +} + +type AddPaths struct { + Config *AddPathsConfig `json:"config,omitempty"` +} + +type AfiSafi struct { + MpGracefulRestart *MpGracefulRestart `json:"mpGracefulRestart,omitempty"` + Config *AfiSafiConfig `json:"config,omitempty"` + AddPaths *AddPaths `json:"addPaths,omitempty"` +} + +func (c BgpPeerSpec) ConverToGoBgpPeer() (*api.Peer, error) { + c.NodeSelector = nil + + jsonBytes, err := json.Marshal(c) + if err != nil { + return nil, err + } + + var result api.Peer + m := jsonpb.Unmarshaler{} + return &result, m.Unmarshal(bytes.NewReader(jsonBytes), &result) +} + +func ConverFromGoBgpPeer(peer *api.Peer) (*BgpPeerSpec, error) { + m := jsonpb.Marshaler{} + jsonStr, err := m.MarshalToString(peer) + if err != nil { + return nil, err + } + + var result BgpPeerSpec + + return &result, json.Unmarshal([]byte(jsonStr), &result) +} + +func ConverStatusFromGoBgpPeer(peer *api.Peer) (NodePeerStatus, error) { + var ( + nodePeerStatus NodePeerStatus + ) + + m := jsonpb.Marshaler{} + jsonStr, err := m.MarshalToString(peer.State) + if err != nil { + return nodePeerStatus, err + } + + err = json.Unmarshal([]byte(jsonStr), &nodePeerStatus.PeerState) + if err != nil { + return nodePeerStatus, err + } + + jsonStr, err = m.MarshalToString(peer.Timers.State) + if err != nil { + return nodePeerStatus, err + } + + err = json.Unmarshal([]byte(jsonStr), &nodePeerStatus.TimersState) + + return nodePeerStatus, err +} + +type BgpPeerSpec struct { + Conf *PeerConf `json:"conf,omitempty"` + Timers *Timers `json:"timers,omitempty"` + Transport *Transport `json:"transport,omitempty"` + GracefulRestart *GracefulRestart `json:"gracefulRestart,omitempty"` + AfiSafis []*AfiSafi `json:"afiSafis,omitempty"` + + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` +} + +// +kubebuilder:object:root=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// BgpPeerList contains a list of BgpPeer +type BgpPeerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BgpPeer `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BgpPeer{}, &BgpPeerList{}) +} diff --git a/api/v1alpha2/eip_types.go b/api/v1alpha2/eip_types.go new file mode 100644 index 000000000..9a1ede27b --- /dev/null +++ b/api/v1alpha2/eip_types.go @@ -0,0 +1,214 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha2 + +import ( + "context" + "fmt" + "math/big" + "net" + "reflect" + "strings" + + "github.com/kubesphere/porter/pkg/constant" + "github.com/kubesphere/porter/pkg/manager/client" + cnet "github.com/projectcalico/libcalico-go/lib/net" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +func (e Eip) IPToOrdinal(ip net.IP) int { + base, size, err := e.GetSize() + if err != nil { + return -1 + } + ipAsInt := cnet.IPToBigInt(cnet.IP{IP: ip}) + baseInt := cnet.IPToBigInt(cnet.IP{IP: base}) + ord := big.NewInt(0).Sub(ipAsInt, baseInt).Int64() + if ord < 0 || ord >= size { + return -1 + } + return int(ord) +} + +func (e Eip) GetSpeakerName() string { + if e.Spec.Protocol == constant.PorterProtocolLayer2 { + return e.Spec.Interface + } + + return constant.PorterProtocolBGP +} + +func (e Eip) GetProtocol() string { + if e.Spec.Protocol == constant.PorterProtocolLayer2 { + return constant.PorterProtocolLayer2 + } + + return constant.PorterProtocolBGP +} + +func (e Eip) GetSize() (net.IP, int64, error) { + ip := net.ParseIP(e.Spec.Address) + if ip != nil { + return ip, 1, nil + } + + _, cidr, err := net.ParseCIDR(e.Spec.Address) + if err == nil { + ones, size := cidr.Mask.Size() + num := 1 << uint(size-ones) + return cidr.IP, int64(num), nil + } + + strs := strings.SplitN(e.Spec.Address, constant.EipRangeSeparator, 2) + if len(strs) != 2 { + return nil, 0, fmt.Errorf("invalid eip address format") + } + base := cnet.ParseIP(strs[0]) + last := cnet.ParseIP(strs[1]) + if base == nil || last == nil { + return nil, 0, fmt.Errorf("invalid eip address format") + } + + ord := big.NewInt(0).Sub(cnet.IPToBigInt(*last), cnet.IPToBigInt(*base)).Int64() + if ord < 0 { + return nil, 0, fmt.Errorf("invalid eip address format") + } + + return base.IP, ord + 1, nil +} + +var _ webhook.Validator = &Eip{} + +// +kubebuilder:webhook:path=/validate-network-kubesphere-io-v1alpha2-eip,mutating=false,failurePolicy=fail,groups=network.kubesphere.io,resources=eips,verbs=create;update;delete,versions=v1alpha2,name=validate.eip.network.kubesphere.io + +func (e Eip) IsOverlap(eip Eip) bool { + base, size, _ := e.GetSize() + + tBase, tSize, _ := eip.GetSize() + if cnet.IPToBigInt(cnet.IP{IP: base}). + Cmp(big.NewInt(0). + Add(cnet.IPToBigInt(cnet.IP{IP: tBase}), big.NewInt(tSize-1))) > 0 || + cnet.IPToBigInt(cnet.IP{IP: tBase}).Cmp(big.NewInt(0). + Add(cnet.IPToBigInt(cnet.IP{IP: base}), big.NewInt(size-1))) > 0 { + return false + } + return true +} + +func (e Eip) ValidateCreate() error { + _, _, err := e.GetSize() + if err != nil { + return err + } + + eips := EipList{} + err = client.Client.List(context.Background(), &eips) + if err != nil { + return err + } + + for _, eip := range eips.Items { + if e.IsOverlap(eip) { + return fmt.Errorf("eip address overlap with %s", eip.Name) + } + } + + if e.Spec.Protocol == constant.PorterProtocolLayer2 { + if e.Spec.Interface == "" { + return fmt.Errorf("field spec.interface should not be empty") + } + } + + return nil +} + +func (e Eip) ValidateUpdate(old runtime.Object) error { + oldE := old.(*Eip) + if !reflect.DeepEqual(e.Spec, oldE.Spec) { + if e.Spec.Disable == oldE.Spec.Disable { + return fmt.Errorf("only allow modify field disable") + } + } + return nil +} + +func (e Eip) ValidateDelete() error { + if e.Status.Usage > 0 { + return fmt.Errorf("eip is using") + } + return nil +} + +func (e Eip) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&e). + Complete() +} + +// EipSpec defines the desired state of EIP +type EipSpec struct { + // +kubebuilder:validation:Required + Address string `json:"address,required"` + // +kubebuilder:validation:Enum=bgp;layer2 + Protocol string `json:"protocol,omitempty"` + Interface string `json:"interface,omitempty"` + Disable bool `json:"disable,omitempty"` + UsingKnownIPs bool `json:"usingKnownIPs,omitempty"` +} + +// EipStatus defines the observed state of EIP +type EipStatus struct { + Occupied bool `json:"occupied,omitempty"` + Usage int `json:"usage,omitempty"` + PoolSize int `json:"poolSize,omitempty"` + Used map[string]string `json:"used,omitempty"` + FirstIP string `json:"firstIP,omitempty"` + LastIP string `json:"lastIP,omitempty"` + Ready bool `json:"ready,omitempty"` + V4 bool `json:"v4,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:name="cidr",type=string,JSONPath=`.spec.address` +// +kubebuilder:printcolumn:name="usage",type=integer,JSONPath=`.status.usage` +// +kubebuilder:printcolumn:name="total",type=integer,JSONPath=`.status.poolSize` +// +kubebuilder:resource:scope=Cluster,categories=networking + +// Eip is the Schema for the eips API +type Eip struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec EipSpec `json:"spec,omitempty"` + Status EipStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true +// EipList contains a list of Eip +type EipList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Eip `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Eip{}, &EipList{}) +} diff --git a/api/v1alpha2/groupversion_info.go b/api/v1alpha2/groupversion_info.go new file mode 100644 index 000000000..febec9b51 --- /dev/null +++ b/api/v1alpha2/groupversion_info.go @@ -0,0 +1,35 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=network.kubesphere.io +// +kubebuilder:storageversion +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "network.kubesphere.io", Version: "v1alpha2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1alpha2/types_test.go b/api/v1alpha2/types_test.go new file mode 100644 index 000000000..7b8ef7a3e --- /dev/null +++ b/api/v1alpha2/types_test.go @@ -0,0 +1,130 @@ +package v1alpha2 + +import ( + "net" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestTypes(t *testing.T) { + RegisterFailHandler(Fail) + log := zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) + RunSpecs(t, "v1alpha2 types Suite") +} + +var _ = Describe("Test eip types", func() { + It("Test GetSize", func() { + e := &Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: EipSpec{ + Address: "192.168.0.1", + }, + Status: EipStatus{}, + } + + base, size, err := e.GetSize() + Expect(base.String()).Should(Equal("192.168.0.1")) + Expect(size).Should(Equal(int64(1))) + Expect(err).ShouldNot(HaveOccurred()) + + e.Spec.Address = "192.168.0.1/24" + base, size, err = e.GetSize() + Expect(base.String()).Should(Equal("192.168.0.0")) + Expect(size).Should(Equal(int64(256))) + Expect(err).ShouldNot(HaveOccurred()) + + e.Spec.Address = "192.168.0.1-192.168.0.100" + base, size, err = e.GetSize() + Expect(base.String()).Should(Equal("192.168.0.1")) + Expect(size).Should(Equal(int64(100))) + Expect(err).ShouldNot(HaveOccurred()) + + e.Spec.Address = "192.168.0.100-192.168.0.1" + base, size, err = e.GetSize() + Expect(err).Should(HaveOccurred()) + + e.Spec.Address = "xxxx" + base, size, err = e.GetSize() + Expect(err).Should(HaveOccurred()) + + e.Spec.Address = "192.168.0.1-192.168.0.100-192.168.0.200" + base, size, err = e.GetSize() + Expect(err).Should(HaveOccurred()) + }) + + It("Test IPToOrdinal", func() { + e := &Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: EipSpec{ + Address: "192.168.0.1-192.168.0.100", + }, + Status: EipStatus{}, + } + + offset := e.IPToOrdinal(net.ParseIP("192.168.0.2")) + Expect(offset).Should(Equal(1)) + + offset = e.IPToOrdinal(net.ParseIP("192.168.0.0")) + Expect(offset).Should(Equal(-1)) + + offset = e.IPToOrdinal(net.ParseIP("192.168.0.101")) + Expect(offset).Should(Equal(-1)) + }) + + It("Test IsOverlap", func() { + e := &Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: EipSpec{ + Address: "192.168.0.100-192.168.0.200", + }, + Status: EipStatus{}, + } + + e2 := Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: EipSpec{ + Address: "192.168.0.1-192.168.0.99", + }, + Status: EipStatus{}, + } + Expect(e.IsOverlap(e2)).Should(BeFalse()) + + e2.Spec.Address = "192.168.0.201-192.168.0.250" + Expect(e.IsOverlap(e2)).Should(BeFalse()) + + e2.Spec.Address = "192.168.0.1-192.168.0.100" + Expect(e.IsOverlap(e2)).Should(BeTrue()) + + e2.Spec.Address = "192.168.0.200-192.168.0.250" + Expect(e.IsOverlap(e2)).Should(BeTrue()) + }) + + It("Test ValidateUpdate", func() { + e := &Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: EipSpec{ + Address: "192.168.0.100-192.168.0.200", + }, + Status: EipStatus{}, + } + + e2 := e.DeepCopy() + e2.Spec.Address = "192.168.0.100" + Expect(e2.ValidateUpdate(e)).Should(HaveOccurred()) + + e2 = e.DeepCopy() + e2.Spec.Disable = true + Expect(e2.ValidateUpdate(e)).ShouldNot(HaveOccurred()) + }) +}) diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go new file mode 100644 index 000000000..c4a1ee683 --- /dev/null +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -0,0 +1,707 @@ +// +build !ignore_autogenerated + +/* +Copyright 2019 The Kubesphere Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddPaths) DeepCopyInto(out *AddPaths) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(AddPathsConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddPaths. +func (in *AddPaths) DeepCopy() *AddPaths { + if in == nil { + return nil + } + out := new(AddPaths) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AddPathsConfig) DeepCopyInto(out *AddPathsConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddPathsConfig. +func (in *AddPathsConfig) DeepCopy() *AddPathsConfig { + if in == nil { + return nil + } + out := new(AddPathsConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AfiSafi) DeepCopyInto(out *AfiSafi) { + *out = *in + if in.MpGracefulRestart != nil { + in, out := &in.MpGracefulRestart, &out.MpGracefulRestart + *out = new(MpGracefulRestart) + (*in).DeepCopyInto(*out) + } + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(AfiSafiConfig) + (*in).DeepCopyInto(*out) + } + if in.AddPaths != nil { + in, out := &in.AddPaths, &out.AddPaths + *out = new(AddPaths) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AfiSafi. +func (in *AfiSafi) DeepCopy() *AfiSafi { + if in == nil { + return nil + } + out := new(AfiSafi) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AfiSafiConfig) DeepCopyInto(out *AfiSafiConfig) { + *out = *in + if in.Family != nil { + in, out := &in.Family, &out.Family + *out = new(Family) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AfiSafiConfig. +func (in *AfiSafiConfig) DeepCopy() *AfiSafiConfig { + if in == nil { + return nil + } + out := new(AfiSafiConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpConf) DeepCopyInto(out *BgpConf) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpConf. +func (in *BgpConf) DeepCopy() *BgpConf { + if in == nil { + return nil + } + out := new(BgpConf) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpConf) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpConfList) DeepCopyInto(out *BgpConfList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpConf, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpConfList. +func (in *BgpConfList) DeepCopy() *BgpConfList { + if in == nil { + return nil + } + out := new(BgpConfList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpConfList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpConfSpec) DeepCopyInto(out *BgpConfSpec) { + *out = *in + if in.ListenAddresses != nil { + in, out := &in.ListenAddresses, &out.ListenAddresses + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Families != nil { + in, out := &in.Families, &out.Families + *out = make([]uint32, len(*in)) + copy(*out, *in) + } + if in.GracefulRestart != nil { + in, out := &in.GracefulRestart, &out.GracefulRestart + *out = new(GracefulRestart) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpConfSpec. +func (in *BgpConfSpec) DeepCopy() *BgpConfSpec { + if in == nil { + return nil + } + out := new(BgpConfSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpConfStatus) DeepCopyInto(out *BgpConfStatus) { + *out = *in + if in.NodesConfStatus != nil { + in, out := &in.NodesConfStatus, &out.NodesConfStatus + *out = make(map[string]NodeConfStatus, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpConfStatus. +func (in *BgpConfStatus) DeepCopy() *BgpConfStatus { + if in == nil { + return nil + } + out := new(BgpConfStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpPeer) DeepCopyInto(out *BgpPeer) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpPeer. +func (in *BgpPeer) DeepCopy() *BgpPeer { + if in == nil { + return nil + } + out := new(BgpPeer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpPeer) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpPeerList) DeepCopyInto(out *BgpPeerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]BgpPeer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpPeerList. +func (in *BgpPeerList) DeepCopy() *BgpPeerList { + if in == nil { + return nil + } + out := new(BgpPeerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BgpPeerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpPeerSpec) DeepCopyInto(out *BgpPeerSpec) { + *out = *in + if in.Conf != nil { + in, out := &in.Conf, &out.Conf + *out = new(PeerConf) + **out = **in + } + if in.Timers != nil { + in, out := &in.Timers, &out.Timers + *out = new(Timers) + (*in).DeepCopyInto(*out) + } + if in.Transport != nil { + in, out := &in.Transport, &out.Transport + *out = new(Transport) + **out = **in + } + if in.GracefulRestart != nil { + in, out := &in.GracefulRestart, &out.GracefulRestart + *out = new(GracefulRestart) + **out = **in + } + if in.AfiSafis != nil { + in, out := &in.AfiSafis, &out.AfiSafis + *out = make([]*AfiSafi, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(AfiSafi) + (*in).DeepCopyInto(*out) + } + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpPeerSpec. +func (in *BgpPeerSpec) DeepCopy() *BgpPeerSpec { + if in == nil { + return nil + } + out := new(BgpPeerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BgpPeerStatus) DeepCopyInto(out *BgpPeerStatus) { + *out = *in + if in.NodesPeerStatus != nil { + in, out := &in.NodesPeerStatus, &out.NodesPeerStatus + *out = make(map[string]NodePeerStatus, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BgpPeerStatus. +func (in *BgpPeerStatus) DeepCopy() *BgpPeerStatus { + if in == nil { + return nil + } + out := new(BgpPeerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Eip) DeepCopyInto(out *Eip) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Eip. +func (in *Eip) DeepCopy() *Eip { + if in == nil { + return nil + } + out := new(Eip) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Eip) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EipList) DeepCopyInto(out *EipList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Eip, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EipList. +func (in *EipList) DeepCopy() *EipList { + if in == nil { + return nil + } + out := new(EipList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EipList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EipSpec) DeepCopyInto(out *EipSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EipSpec. +func (in *EipSpec) DeepCopy() *EipSpec { + if in == nil { + return nil + } + out := new(EipSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EipStatus) DeepCopyInto(out *EipStatus) { + *out = *in + if in.Used != nil { + in, out := &in.Used, &out.Used + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EipStatus. +func (in *EipStatus) DeepCopy() *EipStatus { + if in == nil { + return nil + } + out := new(EipStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Family) DeepCopyInto(out *Family) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Family. +func (in *Family) DeepCopy() *Family { + if in == nil { + return nil + } + out := new(Family) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GracefulRestart) DeepCopyInto(out *GracefulRestart) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GracefulRestart. +func (in *GracefulRestart) DeepCopy() *GracefulRestart { + if in == nil { + return nil + } + out := new(GracefulRestart) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Message) DeepCopyInto(out *Message) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Message. +func (in *Message) DeepCopy() *Message { + if in == nil { + return nil + } + out := new(Message) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Messages) DeepCopyInto(out *Messages) { + *out = *in + if in.Received != nil { + in, out := &in.Received, &out.Received + *out = new(Message) + **out = **in + } + if in.Sent != nil { + in, out := &in.Sent, &out.Sent + *out = new(Message) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Messages. +func (in *Messages) DeepCopy() *Messages { + if in == nil { + return nil + } + out := new(Messages) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MpGracefulRestart) DeepCopyInto(out *MpGracefulRestart) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(MpGracefulRestartConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MpGracefulRestart. +func (in *MpGracefulRestart) DeepCopy() *MpGracefulRestart { + if in == nil { + return nil + } + out := new(MpGracefulRestart) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MpGracefulRestartConfig) DeepCopyInto(out *MpGracefulRestartConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MpGracefulRestartConfig. +func (in *MpGracefulRestartConfig) DeepCopy() *MpGracefulRestartConfig { + if in == nil { + return nil + } + out := new(MpGracefulRestartConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeConfStatus) DeepCopyInto(out *NodeConfStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeConfStatus. +func (in *NodeConfStatus) DeepCopy() *NodeConfStatus { + if in == nil { + return nil + } + out := new(NodeConfStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePeerStatus) DeepCopyInto(out *NodePeerStatus) { + *out = *in + in.PeerState.DeepCopyInto(&out.PeerState) + out.TimersState = in.TimersState +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePeerStatus. +func (in *NodePeerStatus) DeepCopy() *NodePeerStatus { + if in == nil { + return nil + } + out := new(NodePeerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerConf) DeepCopyInto(out *PeerConf) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerConf. +func (in *PeerConf) DeepCopy() *PeerConf { + if in == nil { + return nil + } + out := new(PeerConf) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerState) DeepCopyInto(out *PeerState) { + *out = *in + if in.Messages != nil { + in, out := &in.Messages, &out.Messages + *out = new(Messages) + (*in).DeepCopyInto(*out) + } + if in.Queues != nil { + in, out := &in.Queues, &out.Queues + *out = new(Queues) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerState. +func (in *PeerState) DeepCopy() *PeerState { + if in == nil { + return nil + } + out := new(PeerState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Queues) DeepCopyInto(out *Queues) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Queues. +func (in *Queues) DeepCopy() *Queues { + if in == nil { + return nil + } + out := new(Queues) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Timers) DeepCopyInto(out *Timers) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(TimersConfig) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Timers. +func (in *Timers) DeepCopy() *Timers { + if in == nil { + return nil + } + out := new(Timers) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimersConfig) DeepCopyInto(out *TimersConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimersConfig. +func (in *TimersConfig) DeepCopy() *TimersConfig { + if in == nil { + return nil + } + out := new(TimersConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TimersState) DeepCopyInto(out *TimersState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TimersState. +func (in *TimersState) DeepCopy() *TimersState { + if in == nil { + return nil + } + out := new(TimersState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Transport) DeepCopyInto(out *Transport) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Transport. +func (in *Transport) DeepCopy() *Transport { + if in == nil { + return nil + } + out := new(Transport) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/agent/Dockerfile b/cmd/agent/Dockerfile index 1ba30c646..f1f59d843 100644 --- a/cmd/agent/Dockerfile +++ b/cmd/agent/Dockerfile @@ -1,6 +1,6 @@ FROM alpine WORKDIR / -RUN apk add iptables +RUN apk update && apk add iptables ARG TARGETOS ARG TARGETARCH COPY bin/agent-${TARGETOS}-${TARGETARCH} agent diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 19f941fd3..757421102 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -17,57 +17,59 @@ limitations under the License. package main import ( - "flag" "os" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - "github.com/kubesphere/porter/controllers/eip" + "github.com/kubesphere/porter/pkg/log" + + networkv1alpha2 "github.com/kubesphere/porter/api/v1alpha2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var ( - metricsAddr string - - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() ) func init() { _ = clientgoscheme.AddToScheme(scheme) - _ = networkv1alpha1.AddToScheme(scheme) + _ = networkv1alpha2.AddToScheme(scheme) _ = corev1.AddToScheme(scheme) - flag.StringVar(&metricsAddr, "metrics-addr", ":8081", "The address the metric endpoint binds to.") } func main() { - flag.Parse() - ctrl.SetLogger(zap.Logger(true)) - setupLog.Info("setting up client for agent") + log.InitLog(log.NewOptions()) + + setupLog := ctrl.Log.WithName("setup") + + setupLog.Info("setting up porter agent") mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, + Scheme: scheme, }) if err != nil { - setupLog.Error(err, "unable to start manager") + setupLog.Error(err, "unable to start porter agent") os.Exit(1) } - if err = (&eip.EipReconciler{ - Log: ctrl.Log.WithName("controllers").WithName("Eip"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Eip") - os.Exit(1) - } + mgr.Add(Fake{}) + // Start the Cmd - setupLog.Info("Starting the Cmd.") + setupLog.Info("Starting the porter agent") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "unable to run the manager") + setupLog.Error(err, "unable to run the porter agent") os.Exit(1) } } + +// At the moment, the agent has no tasks to do, but it may +//be needed for future extensions, so it is kept here. +type Fake struct { +} + +func (f Fake) Start(stopCh <-chan struct{}) error { + <-stopCh + return nil +} diff --git a/cmd/manager/app/manager.go b/cmd/manager/app/manager.go index 0b9302060..1bd879ebc 100644 --- a/cmd/manager/app/manager.go +++ b/cmd/manager/app/manager.go @@ -1,21 +1,25 @@ package app import ( + "flag" "fmt" + networkv1alpha2 "github.com/kubesphere/porter/api/v1alpha2" + "github.com/kubesphere/porter/pkg/leader-elector" + clientset "k8s.io/client-go/kubernetes" "net/http" "os" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" "github.com/kubesphere/porter/cmd/manager/app/options" - "github.com/kubesphere/porter/controllers/bgp" - "github.com/kubesphere/porter/controllers/lb" - bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd" - "github.com/kubesphere/porter/pkg/ipam" + "github.com/kubesphere/porter/pkg/constant" + "github.com/kubesphere/porter/pkg/controllers/bgp" + "github.com/kubesphere/porter/pkg/controllers/ipam" + "github.com/kubesphere/porter/pkg/controllers/lb" "github.com/kubesphere/porter/pkg/log" - "github.com/kubesphere/porter/pkg/nettool/iptables" + "github.com/kubesphere/porter/pkg/manager" + "github.com/kubesphere/porter/pkg/speaker" + bgpd "github.com/kubesphere/porter/pkg/speaker/bgp" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" + "github.com/spf13/pflag" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apiserver/pkg/util/term" cliflag "k8s.io/component-base/cli/flag" @@ -42,10 +46,15 @@ func NewPorterManagerCommand() *cobra.Command { } fs := cmd.Flags() + namedFlagSets := s.Flags() for _, f := range namedFlagSets.FlagSets { fs.AddFlagSet(f) } + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + fs.AddFlagSet(pflag.CommandLine) + usageFmt := "Usage:\n %s\n" cols, _, _ := term.TerminalSize(cmd.OutOrStdout()) cmd.SetUsageFunc(func(cmd *cobra.Command) error { @@ -61,14 +70,10 @@ func NewPorterManagerCommand() *cobra.Command { return cmd } -const ( - LeaderElectionID = "porter-manager" -) - func serveReadinessHandler(w http.ResponseWriter, r *http.Request) { if readinessProbe { w.WriteHeader(200) - w.Write([]byte("Hello, World")) + w.Write([]byte("Ready")) } else { w.WriteHeader(500) w.Write([]byte("Not Ready")) @@ -77,66 +82,48 @@ func serveReadinessHandler(w http.ResponseWriter, r *http.Request) { func Run(c *options.PorterManagerOptions) error { log.InitLog(c.LogOptions) + setupLog := ctrl.Log.WithName("setup") - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: c.MetricsAddr, - LeaderElection: c.EnableLeaderElection, - LeaderElectionID: LeaderElectionID, - }) + mgr, err := manager.NewManager(ctrl.GetConfigOrDie(), c.GenericOptions) if err != nil { setupLog.Error(err, "unable to new manager") return err } - bgpServer := bgpserver.NewBgpServer(c.Bgp, ctrl.Log.WithName("bgpServer"), iptables.NewIPTables()) - if err = bgpServer.EnsureNATChain(); err != nil { - setupLog.Error(err, "ensure bgp nat error") + bgpServer := bgpd.NewGoBgpd(c.Bgp) + err = speaker.RegisteSpeaker(constant.PorterProtocolBGP, bgpServer) + if err != nil { + setupLog.Error(err, "unable to register bgp speaker") return err } - ds := ipam.NewDataStore(ctrl.Log.WithName("datastore"), bgpServer) // Setup all Controllers - setupLog.Info("Setting up IPAM") - i := ipam.NewIPAM(ctrl.Log.WithName("IPAM"), ds) - if err = i.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create ipam") + err = ipam.SetupIPAM(mgr) + if err != nil { + setupLog.Error(err, "unable to setup ipam") return err } + networkv1alpha2.Eip{}.SetupWebhookWithManager(mgr) - // Setup bgp Controllers - setupLog.Info("Setting up bgp") - bgpConf := bgp.BgpConfReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("BgpConf"), - BgpServer: bgpServer, - } - if err = bgpConf.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create bgpConf") - return err - } - bgpPeer := bgp.BgpPeerReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("BgpPeer"), - BgpServer: bgpServer, + err = bgp.SetupBgpConfReconciler(bgpServer, mgr) + if err != nil { + setupLog.Error(err, "unable to setup bgpconf") } - if err = bgpPeer.SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create bgpPeer") - return err + + err = bgp.SetupBgpPeerReconciler(bgpServer, mgr) + if err != nil { + setupLog.Error(err, "unable to setup bgppeer") } - setupLog.Info("Setting up lb controller") - if err = (&lb.ServiceReconciler{ - IPAM: i, - Log: ctrl.Log.WithName("controllers").WithName("lb"), - DS: ds, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "lb") + if err = lb.SetupServiceReconciler(mgr); err != nil { + setupLog.Error(err, "unable to setup lb controller") return err } - setupLog.Info("Setting up readiness probe") + k8sClient := clientset.NewForConfigOrDie(ctrl.GetConfigOrDie()) + leader.LeaderElector(k8sClient) + serverMuxA := http.NewServeMux() serverMuxA.HandleFunc("/hello", serveReadinessHandler) go func() { @@ -147,8 +134,6 @@ func Run(c *options.PorterManagerOptions) error { } }() - // Start the Cmd - setupLog.Info("Starting the Cmd.") readinessProbe = true if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "unable to run the manager") @@ -159,11 +144,5 @@ func Run(c *options.PorterManagerOptions) error { } var ( - scheme = runtime.NewScheme() readinessProbe bool ) - -func init() { - _ = corev1.AddToScheme(scheme) - _ = networkv1alpha1.AddToScheme(scheme) -} diff --git a/cmd/manager/app/options/generic.go b/cmd/manager/app/options/generic.go deleted file mode 100644 index ce78587f2..000000000 --- a/cmd/manager/app/options/generic.go +++ /dev/null @@ -1,26 +0,0 @@ -package options - -import ( - "github.com/spf13/pflag" -) - -type GenericOptions struct { - MetricsAddr string - ReadinessAddr string - EnableLeaderElection bool -} - -func NewGenericOptions() *GenericOptions { - return &GenericOptions{ - MetricsAddr: ":8080", - ReadinessAddr: ":8000", - EnableLeaderElection: true, - } -} - -func (options *GenericOptions) AddFlags(fs *pflag.FlagSet) { - fs.StringVar(&options.MetricsAddr, "metrics-addr", options.MetricsAddr, "The address the metric endpoint binds to.") - fs.BoolVar(&options.EnableLeaderElection, "enable-leader-election", options.EnableLeaderElection, "Whether to enable leader "+ - "election. This field should be enabled when porter manager deployed with multiple replicas.") - fs.StringVar(&options.ReadinessAddr, "readiness-addr", options.ReadinessAddr, "The address readinessProbe used") -} diff --git a/cmd/manager/app/options/options.go b/cmd/manager/app/options/options.go index d114f98bb..c525130d8 100644 --- a/cmd/manager/app/options/options.go +++ b/cmd/manager/app/options/options.go @@ -1,22 +1,23 @@ package options import ( - bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd" "github.com/kubesphere/porter/pkg/log" + "github.com/kubesphere/porter/pkg/manager" + "github.com/kubesphere/porter/pkg/speaker/bgp" cliflag "k8s.io/component-base/cli/flag" ) type PorterManagerOptions struct { - Bgp *bgpserver.BgpOptions - *GenericOptions - LogOptions *log.LogOptions + Bgp *bgp.BgpOptions + *manager.GenericOptions + LogOptions *log.Options } func NewPorterManagerOptions() *PorterManagerOptions { return &PorterManagerOptions{ - Bgp: bgpserver.NewBgpOptions(), - GenericOptions: NewGenericOptions(), - LogOptions: log.NewLogOptions(), + Bgp: bgp.NewBgpOptions(), + GenericOptions: manager.NewGenericOptions(), + LogOptions: log.NewOptions(), } } diff --git a/config/crd/bases/network.kubesphere.io_bgpconfs.yaml b/config/crd/bases/network.kubesphere.io_bgpconfs.yaml index 39a614c42..2e9331d16 100644 --- a/config/crd/bases/network.kubesphere.io_bgpconfs.yaml +++ b/config/crd/bases/network.kubesphere.io_bgpconfs.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: bgpconfs.network.kubesphere.io spec: @@ -15,58 +15,145 @@ spec: plural: bgpconfs singular: bgpconf scope: Cluster - validation: - openAPIV3Schema: - description: BgpConf is the Schema for the bgpconfs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: struct for container bgp:config. Configuration parameters relating - to the global BGP router. + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BgpConf is the Schema for the bgpconfs API properties: - as: - description: original -> bgp:as bgp:as's original type is inet:as-number. - Local autonomous system number of the router. Uses the 32-bit as-number - type from the model in RFC 6991. - format: int32 - type: integer - port: - description: original -> gobgp:port - format: int32 - maximum: 65535 - minimum: 1 - type: integer - routerID: - description: original -> bgp:router-id bgp:router-id's original type - is inet:ipv4-address. Router id of the router, expressed as an 32-bit - value, IPv4 address. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string - required: - - as - - port - - routerID + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: struct for container bgp:config. Configuration parameters + relating to the global BGP router. + properties: + as: + description: original -> bgp:as bgp:as's original type is inet:as-number. + Local autonomous system number of the router. Uses the 32-bit as-number + type from the model in RFC 6991. + format: int32 + type: integer + port: + description: original -> gobgp:port + format: int32 + maximum: 65535 + minimum: 1 + type: integer + routerID: + description: original -> bgp:router-id bgp:router-id's original type + is inet:ipv4-address. Router id of the router, expressed as an 32-bit + value, IPv4 address. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + type: string + required: + - as + - port + - routerID + type: object + status: + description: BgpConfStatus defines the observed state of BgpConf + type: object type: object - status: - description: BgpConfStatus defines the observed state of BgpConf + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: BgpConf is the Schema for the bgpconfs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Configuration parameters relating to the global BGP router. + properties: + as: + format: int32 + type: integer + families: + items: + format: int32 + type: integer + type: array + gracefulRestart: + properties: + deferralTime: + format: int32 + type: integer + enabled: + type: boolean + helperOnly: + type: boolean + localRestarting: + type: boolean + longlivedEnabled: + type: boolean + mode: + type: string + notificationEnabled: + type: boolean + peerRestartTime: + format: int32 + type: integer + peerRestarting: + type: boolean + restartTime: + format: int32 + type: integer + staleRoutesTime: + format: int32 + type: integer + type: object + listenAddresses: + items: + type: string + type: array + listenPort: + format: int32 + type: integer + routerId: + type: string + useMultiplePaths: + type: boolean + type: object + status: + description: BgpConfStatus defines the observed state of BgpConf + properties: + nodesConfStatus: + additionalProperties: + properties: + routerId: + type: string + type: object + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: object + type: object type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" diff --git a/config/crd/bases/network.kubesphere.io_bgppeers.yaml b/config/crd/bases/network.kubesphere.io_bgppeers.yaml index ff5ab0e53..6123372bc 100644 --- a/config/crd/bases/network.kubesphere.io_bgppeers.yaml +++ b/config/crd/bases/network.kubesphere.io_bgppeers.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: bgppeers.network.kubesphere.io spec: @@ -15,80 +15,411 @@ spec: plural: bgppeers singular: bgppeer scope: Cluster - validation: - openAPIV3Schema: - description: BgpPeer is the Schema for the bgppeers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BgpPeer is the Schema for the bgppeers API properties: - addPaths: - description: original -> bgp:add-paths Parameters relating to the advertisement - and receipt of multiple paths for a single NLRI (add-paths). + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: properties: - sendMax: - description: original -> bgp:send-max The maximum number of paths - to advertise to neighbors for a single NLRI. - type: integer + addPaths: + description: original -> bgp:add-paths Parameters relating to the + advertisement and receipt of multiple paths for a single NLRI (add-paths). + properties: + sendMax: + description: original -> bgp:send-max The maximum number of paths + to advertise to neighbors for a single NLRI. + type: integer + type: object + config: + description: original -> bgp:neighbor-address original -> bgp:neighbor-config + Configuration parameters relating to the BGP neighbor or group. + properties: + neighborAddress: + description: original -> bgp:neighbor-address bgp:neighbor-address's + original type is inet:ip-address. Address of the BGP peer, either + in IPv4 or IPv6. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + type: string + peerAs: + description: original -> bgp:peer-as bgp:peer-as's original type + is inet:as-number. AS number of the peer. + format: int32 + type: integer + required: + - neighborAddress + - peerAs + type: object + transport: + description: original -> bgp:transport Transport session parameters + for the BGP neighbor or group. + properties: + passiveMode: + description: original -> bgp:passive-mode bgp:passive-mode's original + type is boolean. Wait for peers to issue requests to open a + BGP session, rather than initiating sessions from the local + router. + type: boolean + remotePort: + description: original -> gobgp:remote-port gobgp:remote-port's + original type is inet:port-number. + maximum: 65535 + minimum: 1 + type: integer + type: object + usingPortForward: + type: boolean + type: object + status: + description: BgpPeerStatus defines the observed state of BgpPeer type: object - config: - description: original -> bgp:neighbor-address original -> bgp:neighbor-config - Configuration parameters relating to the BGP neighbor or group. + type: object + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: BgpPeer is the Schema for the bgppeers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: properties: - neighborAddress: - description: original -> bgp:neighbor-address bgp:neighbor-address's - original type is inet:ip-address. Address of the BGP peer, either - in IPv4 or IPv6. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ - type: string - peerAs: - description: original -> bgp:peer-as bgp:peer-as's original type - is inet:as-number. AS number of the peer. - format: int32 - type: integer - required: - - neighborAddress - - peerAs + afiSafis: + items: + properties: + addPaths: + properties: + config: + properties: + receive: + type: boolean + sendMax: + format: int32 + type: integer + type: object + type: object + config: + properties: + enabled: + type: boolean + family: + properties: + afi: + type: string + safi: + type: string + type: object + type: object + mpGracefulRestart: + properties: + config: + properties: + enabled: + type: boolean + type: object + type: object + type: object + type: array + conf: + properties: + adminDown: + type: boolean + allowOwnAs: + format: int32 + type: integer + authPassword: + type: string + description: + type: string + localAs: + format: int32 + type: integer + neighborAddress: + type: string + neighborInterface: + type: string + peerAs: + format: int32 + type: integer + peerGroup: + type: string + peerType: + format: int32 + type: integer + removePrivateAs: + type: string + replacePeerAs: + type: boolean + routeFlapDamping: + type: boolean + sendCommunity: + format: int32 + type: integer + vrf: + type: string + type: object + gracefulRestart: + properties: + deferralTime: + format: int32 + type: integer + enabled: + type: boolean + helperOnly: + type: boolean + localRestarting: + type: boolean + longlivedEnabled: + type: boolean + mode: + type: string + notificationEnabled: + type: boolean + peerRestartTime: + format: int32 + type: integer + peerRestarting: + type: boolean + restartTime: + format: int32 + type: integer + staleRoutesTime: + format: int32 + type: integer + type: object + nodeSelector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + timers: + properties: + config: + description: https://stackoverflow.com/questions/21151765/cannot-unmarshal-string-into-go-value-of-type-int64 + properties: + connectRetry: + type: string + holdTime: + type: string + keepaliveInterval: + type: string + minimumAdvertisementInterval: + type: string + type: object + type: object + transport: + properties: + mtuDiscovery: + description: LocalAddress string `protobuf:"bytes,1,opt,name=local_address,json=localAddress" + json:"localAddress,omitempty"` LocalPort uint32 `protobuf:"varint,2,opt,name=local_port,json=localPort" + json:"localPort,omitempty"` + type: boolean + passiveMode: + type: boolean + remoteAddress: + type: string + remotePort: + format: int32 + type: integer + tcpMss: + format: int32 + type: integer + type: object type: object - transport: - description: original -> bgp:transport Transport session parameters - for the BGP neighbor or group. + status: + description: BgpPeerStatus defines the observed state of BgpPeer properties: - passiveMode: - description: original -> bgp:passive-mode bgp:passive-mode's original - type is boolean. Wait for peers to issue requests to open a BGP - session, rather than initiating sessions from the local router. - type: boolean - remotePort: - description: original -> gobgp:remote-port gobgp:remote-port's original - type is inet:port-number. - maximum: 65535 - minimum: 1 - type: integer + nodesPeerStatus: + additionalProperties: + properties: + peerState: + properties: + adminState: + type: string + authPassword: + type: string + description: + type: string + flops: + format: int32 + type: integer + localAs: + format: int32 + type: integer + messages: + properties: + received: + properties: + discarded: + type: string + keepalive: + type: string + notification: + type: string + open: + type: string + refresh: + type: string + total: + type: string + update: + type: string + withdrawPrefix: + type: string + withdrawUpdate: + type: string + type: object + sent: + properties: + discarded: + type: string + keepalive: + type: string + notification: + type: string + open: + type: string + refresh: + type: string + total: + type: string + update: + type: string + withdrawPrefix: + type: string + withdrawUpdate: + type: string + type: object + type: object + neighborAddress: + type: string + outQ: + format: int32 + type: integer + peerAs: + format: int32 + type: integer + peerGroup: + type: string + peerType: + format: int32 + type: integer + queues: + properties: + input: + format: int32 + type: integer + output: + format: int32 + type: integer + type: object + removePrivateAs: + format: int32 + type: integer + routeFlapDamping: + type: boolean + routerId: + type: string + sendCommunity: + format: int32 + type: integer + sessionState: + type: string + type: object + timersState: + properties: + connectRetry: + type: string + downtime: + type: string + holdTime: + type: string + keepaliveInterval: + type: string + minimumAdvertisementInterval: + type: string + negotiatedHoldTime: + type: string + uptime: + type: string + type: object + type: object + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: object type: object - usingPortForward: - type: boolean type: object - status: - description: BgpPeerStatus defines the observed state of BgpPeer - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" diff --git a/config/crd/bases/network.kubesphere.io_eips.yaml b/config/crd/bases/network.kubesphere.io_eips.yaml index 671f96885..199027393 100644 --- a/config/crd/bases/network.kubesphere.io_eips.yaml +++ b/config/crd/bases/network.kubesphere.io_eips.yaml @@ -1,84 +1,156 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: eips.network.kubesphere.io spec: - additionalPrinterColumns: - - JSONPath: .spec.address - name: cidr - type: string - - JSONPath: .status.usage - name: usage - type: integer - - JSONPath: .status.poolSize - name: total - type: integer group: network.kubesphere.io names: categories: - - ksnet + - networking kind: Eip listKind: EipList plural: eips singular: eip scope: Cluster - subresources: - status: {} - validation: - openAPIV3Schema: - description: Eip is the Schema for the eips API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + versions: + - additionalPrinterColumns: + - jsonPath: .spec.address + name: cidr type: string - metadata: - type: object - spec: - description: EipSpec defines the desired state of EIP + - jsonPath: .status.usage + name: usage + type: integer + - jsonPath: .status.poolSize + name: total + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Eip is the Schema for the eips API properties: - address: - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}((\/([0-9]|[1-2][0-9]|3[0-2]))|(\-([0-9]{1,3}\.){3}[0-9]{1,3}))?$ + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string - disable: - type: boolean - protocol: - enum: - - bgp - - layer2 + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - usingKnownIPs: - type: boolean - required: - - address + metadata: + type: object + spec: + description: EipSpec defines the desired state of EIP + properties: + address: + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}((\/([0-9]|[1-2][0-9]|3[0-2]))|(\-([0-9]{1,3}\.){3}[0-9]{1,3}))?$ + type: string + disable: + type: boolean + protocol: + enum: + - bgp + - layer2 + type: string + usingKnownIPs: + type: boolean + required: + - address + type: object + status: + description: EipStatus defines the observed state of EIP + properties: + occupied: + type: boolean + poolSize: + type: integer + usage: + type: integer + type: object type: object - status: - description: EipStatus defines the observed state of EIP + served: true + storage: false + subresources: + status: { } + - additionalPrinterColumns: + - jsonPath: .spec.address + name: cidr + type: string + - jsonPath: .status.usage + name: usage + type: integer + - jsonPath: .status.poolSize + name: total + type: integer + name: v1alpha2 + schema: + openAPIV3Schema: + description: Eip is the Schema for the eips API properties: - occupied: - type: boolean - poolSize: - type: integer - usage: - type: integer + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EipSpec defines the desired state of EIP + properties: + address: + type: string + disable: + type: boolean + interface: + type: string + protocol: + enum: + - bgp + - layer2 + type: string + usingKnownIPs: + type: boolean + required: + - address + type: object + status: + description: EipStatus defines the observed state of EIP + properties: + firstIP: + type: string + lastIP: + type: string + occupied: + type: boolean + poolSize: + type: integer + ready: + type: boolean + usage: + type: integer + used: + additionalProperties: + type: string + type: object + v4: + type: boolean + type: object type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 8f4e62900..4b84fb162 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,9 +2,9 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/network.kubesphere.io_eips.yaml -- bases/network.kubesphere.io_bgppeers.yaml -- bases/network.kubesphere.io_bgpconfs.yaml + - bases/network.kubesphere.io_eips.yaml + - bases/network.kubesphere.io_bgppeers.yaml + - bases/network.kubesphere.io_bgpconfs.yaml # +kubebuilder:scaffold:crdkustomizeresource #patches: @@ -24,4 +24,4 @@ resources: # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: -- kustomizeconfig.yaml + - kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml index 6f83d9a94..e9cda7124 100644 --- a/config/crd/kustomizeconfig.yaml +++ b/config/crd/kustomizeconfig.yaml @@ -1,17 +1,17 @@ # This file is for teaching kustomize how to substitute name and namespace reference in CRD nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - group: apiextensions.k8s.io - path: spec/conversion/webhookClientConfig/service/name + - kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/name namespace: -- kind: CustomResourceDefinition - group: apiextensions.k8s.io - path: spec/conversion/webhookClientConfig/service/namespace - create: false + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/namespace + create: false varReference: -- path: metadata/annotations + - path: metadata/annotations diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 872cba74d..58d824532 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,9 +1,6 @@ -bases: -- ../crd -- ../rbac -- ../workloads -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml -#- ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager - +resources: + - ns.yaml + - ../crd + - ../rbac + - ../workloads + - ../webhook diff --git a/config/release/ns.yaml b/config/default/ns.yaml similarity index 100% rename from config/release/ns.yaml rename to config/default/ns.yaml diff --git a/config/dev/agent_image_patch.yaml b/config/dev/agent_image_patch.yaml index 1d2dfc08c..8daed9898 100644 --- a/config/dev/agent_image_patch.yaml +++ b/config/dev/agent_image_patch.yaml @@ -2,12 +2,11 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: porter-agent - namespace: system spec: template: spec: containers: - # Change the value of image field below to your controller image URL - - image: kubespheredev/porter-agent:bc7c58a5 - name: porter-agent - imagePullPolicy: Always + # Change the value of image field below to your controller image URL + - image: kubespheredev/porter-agent:bc7c58a5 + name: porter-agent + imagePullPolicy: Always diff --git a/config/dev/cert.yaml b/config/dev/cert.yaml new file mode 100644 index 000000000..bfc5710a9 --- /dev/null +++ b/config/dev/cert.yaml @@ -0,0 +1,114 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: porter-admission +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: porter-admission +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: porter-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: porter-admission +subjects: + - kind: ServiceAccount + name: porter-admission +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: porter-admission +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: porter-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: porter-admission +subjects: + - kind: ServiceAccount + name: porter-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: porter-admission-create +spec: + template: + metadata: + name: porter-admission-create + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.0 + imagePullPolicy: IfNotPresent + args: + - create + - --host=porter-admission,porter-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=porter-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: porter-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: porter-admission-patch +spec: + template: + metadata: + name: porter-admission-patch + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.0 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=porter-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=porter-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: porter-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 \ No newline at end of file diff --git a/config/dev/kustomization.yaml b/config/dev/kustomization.yaml index eddd75bbc..1b5b33de5 100644 --- a/config/dev/kustomization.yaml +++ b/config/dev/kustomization.yaml @@ -1,11 +1,10 @@ -namespace: porter-testns - -bases: - - ../default +namespace: porter-system resources: - - ns.yaml + - ../default + - cert.yaml patchesStrategicMerge: - - agent_image_patch.yaml + # - agent_image_patch.yaml - manager_image_patch.yaml + - nfs_patch.yaml diff --git a/config/dev/manager_image_patch.yaml b/config/dev/manager_image_patch.yaml index 5e5272383..89494703a 100644 --- a/config/dev/manager_image_patch.yaml +++ b/config/dev/manager_image_patch.yaml @@ -2,12 +2,14 @@ apiVersion: apps/v1 kind: Deployment metadata: name: porter-manager - namespace: system spec: template: spec: containers: - # Change the value of image field below to your controller image URL - - image: kubespheredev/porter:bc7c58a5 - name: manager - imagePullPolicy: Always + # Change the value of image field below to your controller image URL + - image: kubespheredev/porter:bc7c58a5 + name: manager + imagePullPolicy: Always + command: + - /bin/sh + tty: true \ No newline at end of file diff --git a/config/dev/nfs_patch.yaml b/config/dev/nfs_patch.yaml new file mode 100644 index 000000000..93ebf4102 --- /dev/null +++ b/config/dev/nfs_patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: porter-manager +spec: + template: + spec: + volumes: + - name: nfs-test + nfs: + path: /mnt/sharedfolder/ #replace here + server: 172.22.0.2 #replace here + containers: + - name: manager + volumeMounts: + - mountPath: /mnt/ #replace here + name: nfs-test \ No newline at end of file diff --git a/config/dev/ns.yaml b/config/dev/ns.yaml deleted file mode 100644 index 1c9c11a50..000000000 --- a/config/dev/ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: porter-testns \ No newline at end of file diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index c887f9f6f..90fb2c61f 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,5 +1,5 @@ resources: -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml + - role.yaml + - role_binding.yaml + - leader_election_role.yaml + - leader_election_role_binding.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index eaa79158f..3238fca9b 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -4,29 +4,29 @@ kind: Role metadata: name: leader-election-role rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - configmaps/status - verbs: - - get - - update - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases/status + verbs: + - get + - update + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index eed16906f..fc79fd71f 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -7,6 +7,5 @@ roleRef: kind: Role name: leader-election-role subjects: -- kind: ServiceAccount - name: default - namespace: system + - kind: ServiceAccount + name: default \ No newline at end of file diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index dc4c97126..1d3d1d29d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,15 +4,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: manager-role + name: porter-manager-role rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - apiGroups: - "" resources: @@ -21,6 +14,14 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update - apiGroups: - "" resources: diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 8f2658702..ce92ed971 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -5,8 +5,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: manager-role + name: porter-manager-role subjects: -- kind: ServiceAccount - name: default - namespace: system + - kind: ServiceAccount + name: default \ No newline at end of file diff --git a/config/release/agent_image_patch.yaml b/config/release/agent_image_patch.yaml index a6f9bd8d7..611ff459f 100644 --- a/config/release/agent_image_patch.yaml +++ b/config/release/agent_image_patch.yaml @@ -2,11 +2,10 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: porter-agent - namespace: system spec: template: spec: containers: - # Change the value of image field below to your controller image URL - - image: kubespheredev/porter-agent:v0.3-dev - name: porter-agent + # Change the value of image field below to your controller image URL + - image: kubespheredev/porter-agent:v0.3-dev + name: porter-agent diff --git a/config/release/kustomization.yaml b/config/release/kustomization.yaml index f561ffd0c..a063a675b 100644 --- a/config/release/kustomization.yaml +++ b/config/release/kustomization.yaml @@ -1,11 +1,8 @@ namespace: porter-system -bases: -- ../default - resources: -- ns.yaml + - ../default patchesStrategicMerge: -- agent_image_patch.yaml -- manager_image_patch.yaml +# - agent_image_patch.yaml + - manager_image_patch.yaml diff --git a/config/release/manager_image_patch.yaml b/config/release/manager_image_patch.yaml index 9a1355eaf..f36834f49 100644 --- a/config/release/manager_image_patch.yaml +++ b/config/release/manager_image_patch.yaml @@ -2,11 +2,17 @@ apiVersion: apps/v1 kind: Deployment metadata: name: porter-manager - namespace: system spec: template: spec: containers: - # Change the value of image field below to your controller image URL - - image: kubespheredev/porter:v0.3-dev - name: manager + # Change the value of image field below to your controller image URL + - image: kubespheredev/porter:v0.3-dev + name: manager + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 100Mi diff --git a/config/samples/network_v1alpha1_bgpconf.yaml b/config/samples/network_v1alpha1_bgpconf.yaml deleted file mode 100644 index 48c136609..000000000 --- a/config/samples/network_v1alpha1_bgpconf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpConf -metadata: - name: bgpconf-sample -spec: - # Add fields here - as : 65000 - routerID : 192.168.0.2 - port: 17900 - diff --git a/config/samples/network_v1alpha1_bgppeer.yaml b/config/samples/network_v1alpha1_bgppeer.yaml deleted file mode 100644 index 57475a43a..000000000 --- a/config/samples/network_v1alpha1_bgppeer.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpPeer -metadata: - name: bgppeer-sample -spec: - # Add fields here - config: - peerAs : 65001 - neighborAddress: 192.168.0.6 - addPaths: - sendMax: 10 diff --git a/config/samples/network_v1alpha1_eip.yaml b/config/samples/network_v1alpha1_eip.yaml deleted file mode 100644 index 430ec0974..000000000 --- a/config/samples/network_v1alpha1_eip.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: network.kubesphere.io/v1alpha1 -kind: Eip -metadata: - name: eip-sample -spec: - address: 139.198.121.228 - disable: false \ No newline at end of file diff --git a/config/samples/network_v1alpha2_bgpconf.yaml b/config/samples/network_v1alpha2_bgpconf.yaml new file mode 100644 index 000000000..0d63e5b76 --- /dev/null +++ b/config/samples/network_v1alpha2_bgpconf.yaml @@ -0,0 +1,13 @@ +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpConf +metadata: + #The porter only recognizes configurations with default names; + #configurations with other names are ignored. + name: default +spec: + as: 50001 + listenPort: 17900 + #Modify the router id as you see fit, if it is not specified + #then the porter will use the node ip as the router id. + routerId: 172.22.0.10 + diff --git a/config/samples/network_v1alpha2_bgppeer.yaml b/config/samples/network_v1alpha2_bgppeer.yaml new file mode 100644 index 000000000..2d1c1e7fd --- /dev/null +++ b/config/samples/network_v1alpha2_bgppeer.yaml @@ -0,0 +1,18 @@ +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + name: bgppeer-sample +spec: + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 + #afiSafis: + # - config: + # family: + # afi: AFI_IP + # safi: SAFI_UNICAST + # enabled: true + # addPaths: + # config: + # sendMax: 10 + diff --git a/config/samples/network_v1alpha2_eip.yaml b/config/samples/network_v1alpha2_eip.yaml new file mode 100644 index 000000000..f7f4f788e --- /dev/null +++ b/config/samples/network_v1alpha2_eip.yaml @@ -0,0 +1,6 @@ +apiVersion: network.kubesphere.io/v1alpha2 +kind: Eip +metadata: + name: eip-sample +spec: + address: 139.198.121.228 \ No newline at end of file diff --git a/config/samples/network_v1alpha2_eip_layer2.yaml b/config/samples/network_v1alpha2_eip_layer2.yaml new file mode 100644 index 000000000..b11b969ff --- /dev/null +++ b/config/samples/network_v1alpha2_eip_layer2.yaml @@ -0,0 +1,9 @@ +apiVersion: network.kubesphere.io/v1alpha2 +kind: Eip +metadata: + name: eip-sample-layer2 +spec: + address: 172.22.0.188-172.22.0.200 + protocol: layer2 + #The interface must be specified when the protocol is layer2. + interface: eth0 \ No newline at end of file diff --git a/config/samples/workload.yaml b/config/samples/workload.yaml new file mode 100644 index 000000000..602739441 --- /dev/null +++ b/config/samples/workload.yaml @@ -0,0 +1,53 @@ +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc-layer2 + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: layer2 +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +--- +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: mylbapp + name: mylbapp +spec: + replicas: 3 + selector: + matchLabels: + app: mylbapp + template: + metadata: + labels: + app: mylbapp + spec: + containers: + - image: nginx:alpine + name: nginx + ports: + - containerPort: 80 \ No newline at end of file diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..43b650add --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - service.yaml + #- manifests.yaml + - webhook.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index e69de29bb..16a6758b3 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -0,0 +1,26 @@ + +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: + - clientConfig: + caBundle: Cg== + service: + name: webhook-service + namespace: system + path: /validate-network-kubesphere-io-v1alpha2-eip + failurePolicy: Fail + name: validate.eip.network.kubesphere.io + rules: + - apiGroups: + - network.kubesphere.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - eips diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..f7417c0f2 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: porter-admission +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: 443 + selector: + control-plane: porter-manager + app: porter-manager \ No newline at end of file diff --git a/config/webhook/webhook.yaml b/config/webhook/webhook.yaml new file mode 100644 index 000000000..ae1ee3937 --- /dev/null +++ b/config/webhook/webhook.yaml @@ -0,0 +1,27 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: porter-admission +webhooks: + - name: validate.eip.network.kubesphere.io + matchPolicy: Equivalent + rules: + - apiGroups: + - network.kubesphere.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE + resources: + - eips + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + namespace: porter-system + name: porter-admission + path: /validate-network-kubesphere-io-v1alpha2-eip \ No newline at end of file diff --git a/config/workloads/agent.yaml b/config/workloads/agent.yaml index 83fb32da0..a25064320 100644 --- a/config/workloads/agent.yaml +++ b/config/workloads/agent.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: porter-agent - namespace: system labels: control-plane: porter-agent controller-tools.k8s.io: "1.0" @@ -27,18 +26,18 @@ spec: cpu: 100m memory: 20Mi env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP securityContext: capabilities: add: - - "NET_ADMIN" + - "NET_ADMIN" hostNetwork: true diff --git a/config/workloads/kustomization.yaml b/config/workloads/kustomization.yaml index 53f59de31..f69f07f09 100644 --- a/config/workloads/kustomization.yaml +++ b/config/workloads/kustomization.yaml @@ -1,3 +1,3 @@ resources: - manager.yaml -- agent.yaml +#- agent.yaml diff --git a/config/workloads/manager.yaml b/config/workloads/manager.yaml index 289d18f46..bf2dff00b 100644 --- a/config/workloads/manager.yaml +++ b/config/workloads/manager.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: Deployment metadata: name: porter-manager - namespace: system labels: control-plane: porter-manager app: porter-manager @@ -17,59 +16,70 @@ spec: control-plane: porter-manager app: porter-manager spec: - hostNetwork: true affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - porter-manager - topologyKey: "kubernetes.io/hostname" + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - porter-manager + topologyKey: "kubernetes.io/hostname" tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - # cloud controller manages should be able to run on masters - - key: "node-role.kubernetes.io/master" - effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + # cloud controller manages should be able to run on masters + - key: "node-role.kubernetes.io/master" + effect: NoSchedule containers: - - command: - - /manager - image: controller:latest - imagePullPolicy: IfNotPresent - name: manager - readinessProbe: - httpGet: - path: /hello - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 3 - securityContext: - capabilities: - add: ["NET_ADMIN", "SYS_TIME"] - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # - name: SECRET_NAME - # value: $(WEBHOOK_SECRET_NAME) - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - # ports: - # - containerPort: 9876 - # name: webhook-server - # protocol: TCP - # volumeMounts: - # - mountPath: /tmp/cert - # name: cert - # readOnly: true + - command: + - /manager + image: controller:latest + imagePullPolicy: IfNotPresent + name: manager + readinessProbe: + httpGet: + path: /hello + port: readness-port + initialDelaySeconds: 5 + periodSeconds: 3 + securityContext: + capabilities: + add: [ "NET_ADMIN", "SYS_TIME" ] + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_ROLE + value: manager + # - name: SECRET_NAME + # value: $(WEBHOOK_SECRET_NAME) + ports: + - containerPort: 8443 + name: webhook-server + protocol: TCP + - containerPort: 8000 + name: readness-port + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /tmp/k8s-webhook-server/serving-certs/ + readOnly: true terminationGracePeriodSeconds: 10 + hostNetwork: true + volumes: + - name: webhook-cert + secret: + secretName: porter-admission + items: + - key: key + path: tls.key + - key: cert + path: tls.crt diff --git a/controllers/bgp/bgpconf_controller.go b/controllers/bgp/bgpconf_controller.go deleted file mode 100644 index f208f7c82..000000000 --- a/controllers/bgp/bgpconf_controller.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2019 The Kubesphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package bgp - -import ( - "context" - "time" - - "github.com/go-logr/logr" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd" - "github.com/kubesphere/porter/pkg/constant" - "github.com/kubesphere/porter/pkg/util" - "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -// BgpConfReconciler reconciles a BgpConf object -type BgpConfReconciler struct { - client.Client - Log logr.Logger - BgpServer *bgpserver.BgpServer -} - -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgpconfs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgpconfs/status,verbs=get;update;patch - -func (r *BgpConfReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - ctx := context.Background() - _ = r.Log.WithValues("bgpconf", req.NamespacedName) - - instance := &networkv1alpha1.BgpConf{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - deleted, err := r.useFinalizerIfNeeded(instance) - if err != nil { - return ctrl.Result{}, err - } - - if deleted { - return ctrl.Result{}, nil - } - - err = r.BgpServer.HandleBgpGlobalConfig(&instance.Spec, false) - - return ctrl.Result{RequeueAfter: time.Second * 60}, err -} - -func (r *BgpConfReconciler) useFinalizerIfNeeded(conf *networkv1alpha1.BgpConf) (bool, error) { - if conf.ObjectMeta.DeletionTimestamp.IsZero() { - if !util.ContainsString(conf.ObjectMeta.Finalizers, constant.FinalizerName) { - conf.ObjectMeta.Finalizers = append(conf.ObjectMeta.Finalizers, constant.FinalizerName) - if err := r.Update(context.Background(), conf); err != nil { - r.Log.Info("Failed to use update to append finalizer to BgpConf", "service", conf.Name) - return false, err - } - r.Log.Info("Append Finalizer to BgpConf", "ServiceName", conf.Name, "Namespace", conf.Namespace) - } - } else { - // The object is being deleted - if util.ContainsString(conf.ObjectMeta.Finalizers, constant.FinalizerName) { - if err := r.BgpServer.HandleBgpGlobalConfig(&conf.Spec, true); err != nil { - return false, err - } - - // remove our finalizer from the list and update it. - conf.ObjectMeta.Finalizers = util.RemoveString(conf.ObjectMeta.Finalizers, constant.FinalizerName) - if err := r.Update(context.Background(), conf); err != nil { - if errors.IsNotFound(err) { - return true, nil - } - return false, err - } - r.Log.Info("Remove Finalizer before service deleted", "ServiceName", conf.Name, "Namespace", conf.Namespace) - return true, nil - } - } - return false, nil -} - -func (r *BgpConfReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&networkv1alpha1.BgpConf{}). - WithEventFilter(predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return true - }, - }).Complete(r) -} diff --git a/controllers/bgp/bgppeer_controller.go b/controllers/bgp/bgppeer_controller.go deleted file mode 100644 index 8f84c98ec..000000000 --- a/controllers/bgp/bgppeer_controller.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2019 The Kubesphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package bgp - -import ( - "context" - "time" - - "github.com/go-logr/logr" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd" - "github.com/kubesphere/porter/pkg/constant" - "github.com/kubesphere/porter/pkg/util" - "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -// BgpPeerReconciler reconciles a BgpPeer object -type BgpPeerReconciler struct { - client.Client - Log logr.Logger - BgpServer *bgpserver.BgpServer -} - -func (r *BgpPeerReconciler) useFinalizerIfNeeded(peer *networkv1alpha1.BgpPeer) (bool, error) { - if peer.ObjectMeta.DeletionTimestamp.IsZero() { - if !util.ContainsString(peer.ObjectMeta.Finalizers, constant.FinalizerName) { - peer.ObjectMeta.Finalizers = append(peer.ObjectMeta.Finalizers, constant.FinalizerName) - if err := r.Update(context.Background(), peer); err != nil { - r.Log.Info("Failed to use update to append finalizer to BgpConf", "service", peer.Name) - return false, err - } - r.Log.Info("Append Finalizer to BgpConf", "ServiceName", peer.Name, "Namespace", peer.Namespace) - } - } else { - // The object is being deleted - if util.ContainsString(peer.ObjectMeta.Finalizers, constant.FinalizerName) { - if err := r.BgpServer.DeletePeer(&peer.Spec); err != nil { - return false, err - } - - // remove our finalizer from the list and update it. - peer.ObjectMeta.Finalizers = util.RemoveString(peer.ObjectMeta.Finalizers, constant.FinalizerName) - if err := r.Update(context.Background(), peer); err != nil { - if errors.IsNotFound(err) { - return true, nil - } - return false, err - } - r.Log.Info("Remove Finalizer before service deleted", "ServiceName", peer.Name, "Namespace", peer.Namespace) - return true, nil - } - } - return false, nil -} - -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgppeers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=bgppeers/status,verbs=get;update;patch - -func (r *BgpPeerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - _ = r.Log.WithValues("bgppeer", req.NamespacedName) - - // your logic here - bgpPeer := &networkv1alpha1.BgpPeer{} - err := r.Get(context.TODO(), req.NamespacedName, bgpPeer) - if err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - deleted, err := r.useFinalizerIfNeeded(bgpPeer) - if err != nil { - return ctrl.Result{}, err - } - - if deleted { - return ctrl.Result{}, nil - } - - err = r.BgpServer.AddOrUpdatePeer(&bgpPeer.Spec) - - return ctrl.Result{RequeueAfter: time.Second * 60}, err -} - -func (r *BgpPeerReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&networkv1alpha1.BgpPeer{}). - WithEventFilter(predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return true - }, - }).Complete(r) -} diff --git a/controllers/eip/eip_controller.go b/controllers/eip/eip_controller.go deleted file mode 100644 index 865b28d90..000000000 --- a/controllers/eip/eip_controller.go +++ /dev/null @@ -1,132 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package eip - -import ( - "context" - "os" - "time" - - "github.com/go-logr/logr" - "github.com/kiali/kiali/log" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - "github.com/kubesphere/porter/pkg/constant" - "github.com/kubesphere/porter/pkg/nettool/iptables" - "github.com/kubesphere/porter/pkg/util" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/tools/record" - "k8s.io/client-go/util/retry" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=eips,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=network.kubesphere.io,resources=eips/status,verbs=get;update;patch -// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch - -// EipReconciler reconciles a Eip object -type EipReconciler struct { - client.Client - Log logr.Logger - record.EventRecorder - iptableExec iptables.IptablesIface -} - -func (r *EipReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - ctx := context.Background() - _ = r.Log.WithValues("eip", req.NamespacedName) - r.Log.Info("----------------Begin to reconclie for eip------------------") - instance := &networkv1alpha1.Eip{} - err := r.Client.Get(ctx, req.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - r.Log.Info("EIP is deleted safely", "name", instance.GetName(), "namespace", instance.GetNamespace()) - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { - err := r.Get(context.TODO(), req.NamespacedName, instance) - if err != nil { - return err - } - _, err = r.useFinalizerIfNeeded(instance) - return err - }) - - if err != nil { - r.Log.Info("Failed to handler finalizer to eip, try again later") - return ctrl.Result{RequeueAfter: time.Second * 10}, err - } - return ctrl.Result{}, nil -} - -func (r *EipReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Client = mgr.GetClient() - r.EventRecorder = mgr.GetEventRecorderFor("eip") - r.iptableExec = iptables.NewIPTables() - return ctrl.NewControllerManagedBy(mgr). - For(&networkv1alpha1.Eip{}). - WithEventFilter(predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - old := e.ObjectOld.(*networkv1alpha1.Eip) - new := e.ObjectNew.(*networkv1alpha1.Eip) - if !e.MetaNew.GetDeletionTimestamp().IsZero() { - return true - } - return old.Status.Occupied != new.Status.Occupied - }, - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - }). - Complete(r) -} - -func (r *EipReconciler) useFinalizerIfNeeded(eip *networkv1alpha1.Eip) (bool, error) { - nodeName := os.Getenv("MY_NODE_NAME") - agentFinalizer := constant.NodeFinalizerName + "/" + nodeName - if eip.ObjectMeta.DeletionTimestamp.IsZero() { - // The object is not being deleted, so if it does not have our finalizer, - // then lets add the finalizer and update the object. - if !util.ContainsString(eip.ObjectMeta.Finalizers, agentFinalizer) { - eip.ObjectMeta.Finalizers = append(eip.ObjectMeta.Finalizers, agentFinalizer) - if err := r.Update(context.Background(), eip); err != nil { - return false, err - } - r.Log.Info("Append Finalizer to eip", "eipName", eip.Name, "Namespace", eip.Namespace) - return false, nil - } - } else { - // The object is being deleted - if util.ContainsString(eip.ObjectMeta.Finalizers, agentFinalizer) { - r.Log.Info("Begin to remove finalizer") - // remove our finalizer from the list and update it. - eip.ObjectMeta.Finalizers = util.RemoveString(eip.ObjectMeta.Finalizers, agentFinalizer) - if err := r.Update(context.Background(), eip); err != nil { - if errors.IsNotFound(err) { - return true, nil - } - return false, err - } - log.Info("Remove Finalizer before eip deleted", "eipName", eip.Name, "Namespace", eip.Namespace) - return true, nil - } - } - return false, nil -} diff --git a/controllers/eip/netutil.go b/controllers/eip/netutil.go deleted file mode 100644 index 290e70239..000000000 --- a/controllers/eip/netutil.go +++ /dev/null @@ -1,63 +0,0 @@ -package eip - -import ( - "github.com/kubesphere/porter/pkg/nettool" -) - -// func (r *EipReconciler) AddRule(instance *networkv1alpha1.Eip) error { -// if instance.Spec.Address != "" { -// rule, err := nettool.NewEIPRule(instance.Spec.Address) -// if err != nil { -// r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address) -// return err -// } -// nodeName := os.Getenv("MY_NODE_NAME") -// if ok, err := rule.IsExist(); err != nil { -// return err -// } else { -// if !ok { -// err = rule.Add() -// if err != nil { -// return err -// } -// r.Event(instance, "Normal", "Rule Created", fmt.Sprintf("Created ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName)) -// } else { -// r.Log.Info("Detect rule in node") -// r.Event(instance, "Normal", "Detect rule in node", fmt.Sprintf("Skipped Creating ip rule for EIP %s in agent %s", instance.Spec.Address, nodeName)) -// } -// } -// } -// return nil -// } - -// func (r *EipReconciler) DeleteRule(instance *networkv1alpha1.Eip) error { -// if instance.Spec.Address != "" { -// rule, err := nettool.NewEIPRule(instance.Spec.Address) -// if err != nil { -// r.Log.Info("Failed to initialize ip rule", "eip", instance.Spec.Address) -// return err -// } -// if ok, err := rule.IsExist(); err != nil { -// return err -// } else { -// if ok { -// err = rule.Delete() -// if err != nil { -// return err -// } -// r.Log.Info("Rule is deleted successfully", "rule", rule.ToAgentRule().String()) -// } else { -// r.Log.Info("Try to delete a non-exist rule", "rule", rule.ToAgentRule().String()) -// } -// } -// } -// return nil -// } - -func (r *EipReconciler) OpenEIPForward(eip string) error { - return nettool.OpenForwardForEIP(r.iptableExec, eip) -} - -func (r *EipReconciler) CloseEIPForward(eip string) error { - return nettool.CloseForwardForEIP(r.iptableExec, eip) -} diff --git a/controllers/lb/controller.go b/controllers/lb/controller.go deleted file mode 100644 index 2a9a1f015..000000000 --- a/controllers/lb/controller.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package lb - -import ( - "context" - "fmt" - "time" - - "github.com/go-logr/logr" - "github.com/kubesphere/porter/pkg/constant" - portererror "github.com/kubesphere/porter/pkg/errors" - "github.com/kubesphere/porter/pkg/ipam" - "github.com/kubesphere/porter/pkg/util" - "github.com/kubesphere/porter/pkg/validate" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch -// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch - -// ServiceReconciler reconciles a Service object -type ServiceReconciler struct { - IPAM *ipam.IPAM - client.Client - Log logr.Logger - record.EventRecorder - - DS *ipam.DataStore -} - -func (r *ServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { - //service - r.Client = mgr.GetClient() - r.EventRecorder = mgr.GetEventRecorderFor("service") - p := predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - if validate.IsTypeLoadBalancer(e.ObjectOld) || validate.IsTypeLoadBalancer(e.ObjectNew) { - if validate.HasPorterLBAnnotation(e.MetaNew.GetAnnotations()) || validate.HasPorterLBAnnotation(e.MetaOld.GetAnnotations()) { - return true - } - } - return false - }, - CreateFunc: func(e event.CreateEvent) bool { - if validate.IsTypeLoadBalancer(e.Object) { - return validate.HasPorterLBAnnotation(e.Meta.GetAnnotations()) - } - return false - }, - } - // Watch for changes to Service - //return ctl.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForObject{}, p) - ctl, err := ctrl.NewControllerManagedBy(mgr).For(&corev1.Service{}).WithEventFilter(p).Named("LBController").Build(r) - if err != nil { - r.Log.Error(err, "Failed to build controller") - return err - } - //endpoints - p = predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - svc := &corev1.Service{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: e.MetaOld.GetNamespace(), Name: e.MetaOld.GetName()}, svc) - if err != nil { - if !errors.IsNotFound(err) { - r.Log.Error(err, "Service missing when watch Endpoints updating") - } - return false - } - if validate.IsTypeLoadBalancer(svc) && validate.HasPorterLBAnnotation(svc.GetAnnotations()) { - old := e.ObjectOld.(*corev1.Endpoints) - new := e.ObjectNew.(*corev1.Endpoints) - return validate.IsNodeChangedWhenEndpointUpdated(old, new) - } - return false - }, - CreateFunc: func(e event.CreateEvent) bool { - svc := &corev1.Service{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: e.Meta.GetNamespace(), Name: e.Meta.GetName()}, svc) - if err != nil { - if !errors.IsNotFound(err) { - r.Log.Error(err, "Something wrong when watch Endpoints creating") - } - return false - } - if validate.IsTypeLoadBalancer(svc) { - return validate.HasPorterLBAnnotation(svc.GetAnnotations()) - } - return false - }, - } - return ctl.Watch(&source.Kind{Type: &corev1.Endpoints{}}, &handler.EnqueueRequestForObject{}, p) -} - -func (r *ServiceReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - log := r.Log.WithValues("porter", req.NamespacedName) - - svc := &corev1.Service{} - err := r.Get(context.TODO(), req.NamespacedName, svc) - if err != nil { - if errors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - deleted, err := r.useFinalizerIfNeeded(svc) - if err == nil { - if deleted { - return ctrl.Result{}, nil - } - err = r.createLB(svc) - } - - if err != nil { - switch portererror.ReasonForError(err) { - case portererror.EIPNotExist: - r.Event(svc, corev1.EventTypeWarning, "LB Created", "Cann't assign EIP, please check eip and protocol") - return ctrl.Result{}, r.updateService(svc, r.findEIP(svc), true) - case portererror.ParaInvalidError: - return ctrl.Result{}, nil - default: - if errors.IsNotFound(err) { - log.Info("Maybe sevice has been deleted, skipping reconciling") - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - } - - return ctrl.Result{RequeueAfter: time.Second * 60}, nil -} - -func (r *ServiceReconciler) useFinalizerIfNeeded(serv *corev1.Service) (bool, error) { - if serv.ObjectMeta.DeletionTimestamp.IsZero() { - // The object is not being deleted, so if it does not have our finalizer, - // then lets add the finalizer and update the object. - // double check before appending finalizer ref: https://github.com/kubesphere/porter/issues/43 - if !validate.HasPorterLBAnnotation(serv.GetAnnotations()) { - r.Log.Error(fmt.Errorf("service does not have porter annotation"), "Watching filter seems not take affect") - return true, nil - } - - if !util.ContainsString(serv.ObjectMeta.Finalizers, constant.FinalizerName) { - controllerutil.AddFinalizer(serv, constant.FinalizerName) - if err := r.Update(context.Background(), serv); err != nil { - r.Log.Info("Failed to use update to append finalizer to service", "service", serv.Name) - return false, err - } - r.Log.Info("Append Finalizer to service", "ServiceName", serv.Name, "Namespace", serv.Namespace) - } - } else { - // The object is being deleted - if util.ContainsString(serv.ObjectMeta.Finalizers, constant.FinalizerName) { - // our finalizer is present, so lets handle our external dependency - if err := r.deleteLB(serv); err != nil { - // if fail to delete the external dependency here, return with error - // so that it can be retried - return false, err - } - // remove our finalizer from the list and update it. - controllerutil.RemoveFinalizer(serv, constant.FinalizerName) - if err := r.Update(context.Background(), serv); err != nil { - if errors.IsNotFound(err) { - return true, nil - } - return false, err - } - r.Log.Info("Remove Finalizer before service deleted", "ServiceName", serv.Name, "Namespace", serv.Namespace) - return true, nil - } - } - return false, nil -} diff --git a/controllers/lb/lb.go b/controllers/lb/lb.go deleted file mode 100644 index 370ba4ad8..000000000 --- a/controllers/lb/lb.go +++ /dev/null @@ -1,157 +0,0 @@ -package lb - -import ( - "context" - "fmt" - "github.com/kubesphere/porter/pkg/constant" - portererror "github.com/kubesphere/porter/pkg/errors" - "github.com/kubesphere/porter/pkg/kubeutil" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" -) - -func (r *ServiceReconciler) findEIP(svc *corev1.Service) string { - if svc.Annotations != nil { - if ip, ok := svc.Annotations[constant.PorterEIPAnnotationKey]; ok { - return ip - } - } - - if svc.Spec.LoadBalancerIP != "" { - return svc.Spec.LoadBalancerIP - } - - return "" -} - -func (r *ServiceReconciler) ensureEIP(serv *corev1.Service, foundOrError bool) (string, error) { - if ip := r.findEIP(serv); ip != "" { - status := r.DS.GetEIPStatus(ip) - if !status.Exist { - return "", portererror.PorterError{Code: portererror.EIPNotExist} - } - if !status.Used { - r.Log.Info("Service has eip but not in pool", "Service", serv.Name, "eip", ip) - _, err := r.DS.AssignSpecifyIP(ip, serv.Annotations[constant.PorterProtocolAnnotationKey], serv.Name, serv.Namespace) - if err != nil { - r.Log.Info("Failed to mark eip as used", "eip", ip) - return "", err - } - } - return ip, nil - } - - if foundOrError { - return "", portererror.PorterError{Code: portererror.EIPNotExist} - } - - resp, err := r.DS.AssignIP(serv.Name, serv.Namespace, serv.Annotations[constant.PorterProtocolAnnotationKey]) - if err != nil { - r.Log.Error(nil, "Failed to get an ip from pool") - return "", err - } - return resp.Address, nil -} - -func (r *ServiceReconciler) createLB(serv *corev1.Service) error { - nexthops, err := r.getServiceNodesIP(serv) - if err != nil { - if errors.IsNotFound(err) { - r.Log.Info("Detect no available endpoints now") - return nil - } - r.Log.Error(nil, "Failed to get ip of nodes where endpoints locate in") - return err - } - - ip, err := r.ensureEIP(serv, false) - if err != nil { - return err - } - - if err = r.updateService(serv, ip, false); err != nil { - return err - } - - err = r.DS.SetBalancer(ip, nexthops) - if err == nil { - for _, nexthop := range nexthops { - r.Log.Info("Add Route to ", "ip", nexthop) - } - } - - r.Log.Info(fmt.Sprintf("Pls visit %s:%d to check it out", ip, serv.Spec.Ports[0].Port)) - return nil -} - -func (r *ServiceReconciler) updateService(serv *corev1.Service, ip string, delete bool) error { - log := r.Log.WithValues("ip", ip, "namespace", serv.Namespace, "name", serv.Name) - - found := false - var tmpIngress []corev1.LoadBalancerIngress - for _, item := range serv.Status.LoadBalancer.Ingress { - if item.IP == ip { - found = true - if !delete { - break - } else { - continue - } - } - tmpIngress = append(tmpIngress, item) - } - - if found && !delete { - return nil - } - - if !delete { - log.V(2).Info("Updating service status & metadata") - serv.Status.LoadBalancer.Ingress = append(serv.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{ - IP: ip, - }) - - if serv.Annotations == nil { - serv.Annotations = make(map[string]string) - } - serv.Annotations[constant.PorterEIPAnnotationKey] = ip - } else { - log.V(2).Info("remove ingress from service") - serv.Status.LoadBalancer.Ingress = tmpIngress - } - - if err := r.Status().Update(context.TODO(), serv); err != nil { - r.DS.UnassignIP(ip) - return err - } else if delete { - return nil - } - - r.Event(serv, corev1.EventTypeNormal, "LB Created", fmt.Sprintf("Successfully assign EIP %s", ip)) - return nil -} - -func (r *ServiceReconciler) deleteLB(serv *corev1.Service) error { - log := r.Log.WithValues("namespace", serv.Namespace, "name", serv.Name) - ip, err := r.ensureEIP(serv, true) - if err != nil { - log.Info("EIP not exist, so we can delete service safely") - return nil - } - - if err = r.DS.DelBalancer(ip); err != nil { - log.Info("Failed to delete router on service", "ip", ip) - return err - } - - if err = r.DS.UnassignIP(ip); err != nil { - log.Info("Failed to revoke ip on service", "ip", ip) - return err - } - - return nil -} - -func (r *ServiceReconciler) getServiceNodesIP(serv *corev1.Service) ([]string, error) { - return kubeutil.GetServiceNodesIP(r.Client, serv) -} diff --git a/controllers/suite_test.go b/controllers/suite_test.go deleted file mode 100644 index 329cef70f..000000000 --- a/controllers/suite_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var cfg *rest.Config -var k8sClient client.Client -var testEnv *envtest.Environment - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecsWithDefaultAndCustomReporters(t, - "Controller Suite", - []Reporter{printer.NewlineReporter{}}) -} - -var _ = BeforeSuite(func(done Done) { - logf.SetLogger(zap.LoggerTo(GinkgoWriter, true)) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - } - - var err error - cfg, err = testEnv.Start() - Expect(err).ToNot(HaveOccurred()) - Expect(cfg).ToNot(BeNil()) - - err = networkv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - err = corev1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - // +kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).ToNot(HaveOccurred()) - Expect(k8sClient).ToNot(BeNil()) - - close(done) -}, 60) - -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).ToNot(HaveOccurred()) -}) diff --git a/deploy/porter.yaml b/deploy/porter.yaml index 99f574074..0bc641728 100644 --- a/deploy/porter.yaml +++ b/deploy/porter.yaml @@ -3,11 +3,11 @@ kind: Namespace metadata: name: porter-system --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: bgpconfs.network.kubesphere.io spec: @@ -18,58 +18,142 @@ spec: plural: bgpconfs singular: bgpconf scope: Cluster - validation: - openAPIV3Schema: - description: BgpConf is the Schema for the bgpconfs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: struct for container bgp:config. Configuration parameters relating - to the global BGP router. + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BgpConf is the Schema for the bgpconfs API properties: - as: - description: original -> bgp:as bgp:as's original type is inet:as-number. - Local autonomous system number of the router. Uses the 32-bit as-number - type from the model in RFC 6991. - format: int32 - type: integer - port: - description: original -> gobgp:port - format: int32 - maximum: 65535 - minimum: 1 - type: integer - routerID: - description: original -> bgp:router-id bgp:router-id's original type - is inet:ipv4-address. Router id of the router, expressed as an 32-bit - value, IPv4 address. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - required: - - as - - port - - routerID + metadata: + type: object + spec: + description: struct for container bgp:config. Configuration parameters + relating to the global BGP router. + properties: + as: + description: original -> bgp:as bgp:as's original type is inet:as-number. + Local autonomous system number of the router. Uses the 32-bit as-number + type from the model in RFC 6991. + format: int32 + type: integer + port: + description: original -> gobgp:port + format: int32 + maximum: 65535 + minimum: 1 + type: integer + routerID: + description: original -> bgp:router-id bgp:router-id's original type + is inet:ipv4-address. Router id of the router, expressed as an 32-bit + value, IPv4 address. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + type: string + required: + - as + - port + - routerID + type: object + status: + description: BgpConfStatus defines the observed state of BgpConf + type: object type: object - status: - description: BgpConfStatus defines the observed state of BgpConf + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: BgpConf is the Schema for the bgpconfs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Configuration parameters relating to the global BGP router. + properties: + as: + format: int32 + type: integer + families: + items: + format: int32 + type: integer + type: array + gracefulRestart: + properties: + deferralTime: + format: int32 + type: integer + enabled: + type: boolean + helperOnly: + type: boolean + localRestarting: + type: boolean + longlivedEnabled: + type: boolean + mode: + type: string + notificationEnabled: + type: boolean + peerRestartTime: + format: int32 + type: integer + peerRestarting: + type: boolean + restartTime: + format: int32 + type: integer + staleRoutesTime: + format: int32 + type: integer + type: object + listenAddresses: + items: + type: string + type: array + listenPort: + format: int32 + type: integer + routerId: + type: string + useMultiplePaths: + type: boolean + type: object + status: + description: BgpConfStatus defines the observed state of BgpConf + properties: + nodesConfStatus: + additionalProperties: + properties: + routerId: + type: string + type: object + type: object + type: object type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" @@ -77,11 +161,11 @@ status: conditions: [] storedVersions: [] --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: bgppeers.network.kubesphere.io spec: @@ -92,80 +176,408 @@ spec: plural: bgppeers singular: bgppeer scope: Cluster - validation: - openAPIV3Schema: - description: BgpPeer is the Schema for the bgppeers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BgpPeer is the Schema for the bgppeers API properties: - addPaths: - description: original -> bgp:add-paths Parameters relating to the advertisement - and receipt of multiple paths for a single NLRI (add-paths). + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: properties: - sendMax: - description: original -> bgp:send-max The maximum number of paths - to advertise to neighbors for a single NLRI. - type: integer + addPaths: + description: original -> bgp:add-paths Parameters relating to the + advertisement and receipt of multiple paths for a single NLRI (add-paths). + properties: + sendMax: + description: original -> bgp:send-max The maximum number of paths + to advertise to neighbors for a single NLRI. + type: integer + type: object + config: + description: original -> bgp:neighbor-address original -> bgp:neighbor-config + Configuration parameters relating to the BGP neighbor or group. + properties: + neighborAddress: + description: original -> bgp:neighbor-address bgp:neighbor-address's + original type is inet:ip-address. Address of the BGP peer, either + in IPv4 or IPv6. + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ + type: string + peerAs: + description: original -> bgp:peer-as bgp:peer-as's original type + is inet:as-number. AS number of the peer. + format: int32 + type: integer + required: + - neighborAddress + - peerAs + type: object + transport: + description: original -> bgp:transport Transport session parameters + for the BGP neighbor or group. + properties: + passiveMode: + description: original -> bgp:passive-mode bgp:passive-mode's original + type is boolean. Wait for peers to issue requests to open a + BGP session, rather than initiating sessions from the local + router. + type: boolean + remotePort: + description: original -> gobgp:remote-port gobgp:remote-port's + original type is inet:port-number. + maximum: 65535 + minimum: 1 + type: integer + type: object + usingPortForward: + type: boolean + type: object + status: + description: BgpPeerStatus defines the observed state of BgpPeer type: object - config: - description: original -> bgp:neighbor-address original -> bgp:neighbor-config - Configuration parameters relating to the BGP neighbor or group. + type: object + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: BgpPeer is the Schema for the bgppeers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: properties: - neighborAddress: - description: original -> bgp:neighbor-address bgp:neighbor-address's - original type is inet:ip-address. Address of the BGP peer, either - in IPv4 or IPv6. - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}$ - type: string - peerAs: - description: original -> bgp:peer-as bgp:peer-as's original type - is inet:as-number. AS number of the peer. - format: int32 - type: integer - required: - - neighborAddress - - peerAs + afiSafis: + items: + properties: + addPaths: + properties: + config: + properties: + receive: + type: boolean + sendMax: + format: int32 + type: integer + type: object + type: object + config: + properties: + enabled: + type: boolean + family: + properties: + afi: + type: string + safi: + type: string + type: object + type: object + mpGracefulRestart: + properties: + config: + properties: + enabled: + type: boolean + type: object + type: object + type: object + type: array + conf: + properties: + adminDown: + type: boolean + allowOwnAs: + format: int32 + type: integer + authPassword: + type: string + description: + type: string + localAs: + format: int32 + type: integer + neighborAddress: + type: string + neighborInterface: + type: string + peerAs: + format: int32 + type: integer + peerGroup: + type: string + peerType: + format: int32 + type: integer + removePrivateAs: + type: string + replacePeerAs: + type: boolean + routeFlapDamping: + type: boolean + sendCommunity: + format: int32 + type: integer + vrf: + type: string + type: object + gracefulRestart: + properties: + deferralTime: + format: int32 + type: integer + enabled: + type: boolean + helperOnly: + type: boolean + localRestarting: + type: boolean + longlivedEnabled: + type: boolean + mode: + type: string + notificationEnabled: + type: boolean + peerRestartTime: + format: int32 + type: integer + peerRestarting: + type: boolean + restartTime: + format: int32 + type: integer + staleRoutesTime: + format: int32 + type: integer + type: object + nodeSelector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + timers: + properties: + config: + description: https://stackoverflow.com/questions/21151765/cannot-unmarshal-string-into-go-value-of-type-int64 + properties: + connectRetry: + type: string + holdTime: + type: string + keepaliveInterval: + type: string + minimumAdvertisementInterval: + type: string + type: object + type: object + transport: + properties: + mtuDiscovery: + type: boolean + passiveMode: + type: boolean + remoteAddress: + type: string + remotePort: + format: int32 + type: integer + tcpMss: + format: int32 + type: integer + type: object type: object - transport: - description: original -> bgp:transport Transport session parameters - for the BGP neighbor or group. + status: + description: BgpPeerStatus defines the observed state of BgpPeer properties: - passiveMode: - description: original -> bgp:passive-mode bgp:passive-mode's original - type is boolean. Wait for peers to issue requests to open a BGP - session, rather than initiating sessions from the local router. - type: boolean - remotePort: - description: original -> gobgp:remote-port gobgp:remote-port's original - type is inet:port-number. - maximum: 65535 - minimum: 1 - type: integer + nodesPeerStatus: + additionalProperties: + properties: + peerState: + properties: + adminState: + type: string + authPassword: + type: string + description: + type: string + flops: + format: int32 + type: integer + localAs: + format: int32 + type: integer + messages: + properties: + received: + properties: + discarded: + type: string + keepalive: + type: string + notification: + type: string + open: + type: string + refresh: + type: string + total: + type: string + update: + type: string + withdrawPrefix: + type: string + withdrawUpdate: + type: string + type: object + sent: + properties: + discarded: + type: string + keepalive: + type: string + notification: + type: string + open: + type: string + refresh: + type: string + total: + type: string + update: + type: string + withdrawPrefix: + type: string + withdrawUpdate: + type: string + type: object + type: object + neighborAddress: + type: string + outQ: + format: int32 + type: integer + peerAs: + format: int32 + type: integer + peerGroup: + type: string + peerType: + format: int32 + type: integer + queues: + properties: + input: + format: int32 + type: integer + output: + format: int32 + type: integer + type: object + removePrivateAs: + format: int32 + type: integer + routeFlapDamping: + type: boolean + routerId: + type: string + sendCommunity: + format: int32 + type: integer + sessionState: + type: string + type: object + timersState: + properties: + connectRetry: + type: string + downtime: + type: string + holdTime: + type: string + keepaliveInterval: + type: string + minimumAdvertisementInterval: + type: string + negotiatedHoldTime: + type: string + uptime: + type: string + type: object + type: object + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: object type: object - usingPortForward: - type: boolean - type: object - status: - description: BgpPeerStatus defines the observed state of BgpPeer type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" @@ -173,91 +585,163 @@ status: conditions: [] storedVersions: [] --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.5 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: eips.network.kubesphere.io spec: - additionalPrinterColumns: - - JSONPath: .spec.address - name: cidr - type: string - - JSONPath: .status.usage - name: usage - type: integer - - JSONPath: .status.poolSize - name: total - type: integer group: network.kubesphere.io names: categories: - - ksnet + - networking kind: Eip listKind: EipList plural: eips singular: eip scope: Cluster - subresources: - status: {} - validation: - openAPIV3Schema: - description: Eip is the Schema for the eips API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + versions: + - additionalPrinterColumns: + - jsonPath: .spec.address + name: cidr type: string - metadata: - type: object - spec: - description: EipSpec defines the desired state of EIP + - jsonPath: .status.usage + name: usage + type: integer + - jsonPath: .status.poolSize + name: total + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: Eip is the Schema for the eips API properties: - address: - pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}((\/([0-9]|[1-2][0-9]|3[0-2]))|(\-([0-9]{1,3}\.){3}[0-9]{1,3}))?$ + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string - disable: - type: boolean - protocol: - enum: - - bgp - - layer2 + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - usingKnownIPs: - type: boolean - required: - - address + metadata: + type: object + spec: + description: EipSpec defines the desired state of EIP + properties: + address: + pattern: ^([0-9]{1,3}\.){3}[0-9]{1,3}((\/([0-9]|[1-2][0-9]|3[0-2]))|(\-([0-9]{1,3}\.){3}[0-9]{1,3}))?$ + type: string + disable: + type: boolean + protocol: + enum: + - bgp + - layer2 + type: string + usingKnownIPs: + type: boolean + required: + - address + type: object + status: + description: EipStatus defines the observed state of EIP + properties: + occupied: + type: boolean + poolSize: + type: integer + usage: + type: integer + type: object type: object - status: - description: EipStatus defines the observed state of EIP + served: true + storage: false + subresources: + status: { } + - additionalPrinterColumns: + - jsonPath: .spec.address + name: cidr + type: string + - jsonPath: .status.usage + name: usage + type: integer + - jsonPath: .status.poolSize + name: total + type: integer + name: v1alpha2 + schema: + openAPIV3Schema: + description: Eip is the Schema for the eips API properties: - occupied: - type: boolean - poolSize: - type: integer - usage: - type: integer + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: EipSpec defines the desired state of EIP + properties: + address: + type: string + disable: + type: boolean + interface: + type: string + protocol: + enum: + - bgp + - layer2 + type: string + usingKnownIPs: + type: boolean + required: + - address + type: object + status: + description: EipStatus defines the observed state of EIP + properties: + firstIP: + type: string + lastIP: + type: string + occupied: + type: boolean + poolSize: + type: integer + ready: + type: boolean + usage: + type: integer + used: + additionalProperties: + type: string + type: object + v4: + type: boolean + type: object type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 - served: true - storage: true + served: true + storage: true + subresources: + status: { } status: acceptedNames: kind: "" plural: "" - conditions: [] - storedVersions: [] + conditions: [ ] + storedVersions: [ ] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -265,75 +749,76 @@ metadata: name: leader-election-role namespace: porter-system rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - configmaps/status - verbs: - - get - - update - - patch -- apiGroups: - - "" - resources: - - events - verbs: - - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases/status + verbs: + - get + - update + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null - name: manager-role + name: porter-manager-role rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: - "" resources: - services/status @@ -412,9 +897,9 @@ roleRef: kind: Role name: leader-election-role subjects: -- kind: ServiceAccount - name: default - namespace: porter-system + - kind: ServiceAccount + name: default + namespace: porter-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -423,11 +908,26 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: manager-role + name: porter-manager-role subjects: -- kind: ServiceAccount - name: default + - kind: ServiceAccount + name: default + namespace: porter-system +--- +apiVersion: v1 +kind: Service +metadata: + name: porter-admission namespace: porter-system +spec: + ports: + - name: https-webhook + port: 443 + targetPort: 443 + selector: + app: porter-manager + control-plane: porter-manager + type: ClusterIP --- apiVersion: apps/v1 kind: Deployment @@ -460,80 +960,91 @@ spec: topologyKey: kubernetes.io/hostname containers: - command: - - /manager + - /manager env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: kubespheredev/porter:v0.3-dev + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_ROLE + value: manager + image: kubespheredev/porter:v0.4 imagePullPolicy: IfNotPresent name: manager + ports: + - containerPort: 8443 + name: webhook-server + protocol: TCP + - containerPort: 8000 + name: readness-port + protocol: TCP readinessProbe: httpGet: path: /hello - port: 8000 + port: readness-port initialDelaySeconds: 5 periodSeconds: 3 resources: limits: cpu: 100m - memory: 30Mi + memory: 300Mi requests: cpu: 100m - memory: 20Mi + memory: 100Mi securityContext: capabilities: add: - - NET_ADMIN - - SYS_TIME + - NET_ADMIN + - SYS_TIME + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs/ + name: webhook-cert + readOnly: true hostNetwork: true terminationGracePeriodSeconds: 10 tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - name: webhook-cert + secret: + items: + - key: key + path: tls.key + - key: cert + path: tls.crt + secretName: porter-admission --- -apiVersion: apps/v1 -kind: DaemonSet +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration metadata: - labels: - control-plane: porter-agent - controller-tools.k8s.io: "1.0" - name: porter-agent - namespace: porter-system -spec: - selector: - matchLabels: - app: porter-agent - template: - metadata: - labels: - app: porter-agent - spec: - containers: - - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - image: kubespheredev/porter-agent:v0.3-dev - imagePullPolicy: IfNotPresent - name: porter-agent + name: porter-admission +webhooks: + - admissionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: porter-admission + namespace: porter-system + path: /validate-network-kubesphere-io-v1alpha2-eip + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.eip.network.kubesphere.io + rules: + - apiGroups: + - network.kubesphere.io + apiVersions: + - v1alpha2 + operations: + - CREATE + - UPDATE resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - securityContext: - capabilities: - add: - - NET_ADMIN - hostNetwork: true + - eips + sideEffects: None diff --git a/doc/bgp_config.md b/doc/bgp_config.md index 1fcd34920..6bcea4bab 100644 --- a/doc/bgp_config.md +++ b/doc/bgp_config.md @@ -1,49 +1,113 @@ -# BGP Configuration +# Introduction to BGP Config > English | [中文](zh/bgp_config.md) -Porter uses [gobgp](https://github.com/osrg/gobgp) to exchange routing information with external routers. There are not many parameters currently used, the following configuration introduces how to configure the BGP server used by Porter. +Porter uses [gobgp](https://github.com/osrg/gobgp) to establish a BGP connection with an external router for route publishing. + +Porter provides two CRDs, BgpConf and BgpPeer, for configuring gobgp respectively. The CRDs are defined in the reference [API for gobgp] (https://github.com/osrg/gobgp/blob/master/api/gobgp.pb.go), which can be used as follows Reference [GoBGP as a Go Native BGP library](https://github.com/osrg/gobgp/blob/master/docs/sources/lib.md) + +## BgpConf + +BgpConf is used to configure the global configuration of gobgp, so only one of these will work, and Porter currently only recognizes configurations with name `default`. + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpConf +metadata: + #The porter only recognizes configurations with default names; + #configurations with other names are ignored. + name: default +spec: + as: 50001 + listenPort: 17900 + #Modify the router id as you see fit, if it is not specified + #then the porter will use the node ip as the router id. + routerId: 172.22.0.10 +``` + +1. `as` is the number of Autonomous System, which must be different from the Autonomous System of the connected routers, the same will cause the route to be incorrectly transmitted. +2. `routerId` denotes the cluster's Id, usually taking the IP of the master NIC of the Kubernetes master node. If you don't specify it, Porter will select the first IP of the node as the routerId. +3. `listenPort` is the port on which gobgp listens, which defaults to 179. Since Calico also uses BGP and occupies port 179, a different port must be specified here. + +### Specify gobgp to listen to IP addresses + +Specify the IP address that gobgp listens to via `ListenAddresses`. -## Global configuration ```yaml -apiVersion: network.kubesphere.io/v1alpha1 +apiVersion: network.kubesphere.io/v1alpha2 kind: BgpConf metadata: - name: bgpconf-sample + #The porter only recognizes configurations with default names; + #configurations with other names are ignored. + name: default spec: - # Add fields here - as : 65000 - routerID : 192.168.0.2 - port: 17900 + as: 50001 + listenPort: 17900 + #Modify the router id as you see fit, if it is not specified + #then the porter will use the node ip as the router id. + routerId: 172.22.0.10 + ListenAddresses: + - 172.22.0.10 ``` -1. `as` is the number of Autonomous System, and it must be different from the AS number of the router. -2. `routeID` is the ID of the cluster. In general, we use the IP of the k8s master node. -3. `port` is the port that gobgp listens to, the default is 179. Because calico also uses BGP and listens the port 179, it is necessary to specify a different port here. If the router does not support ports other than 179, you need to enable port forwarding on the node where the port is located to map 179 to other port. +## BgpPeer + +BgpPeer is used to configure gobgp's neighbor, which can exist in multiple locations, depending on your network environment. -## Configuring BGP Peers -> BGP peers and neighbors mean the same thing. You can add one or more neighbors. ```yaml -apiVersion: network.kubesphere.io/v1alpha1 +apiVersion: network.kubesphere.io/v1alpha2 kind: BgpPeer metadata: name: bgppeer-sample spec: - # Add fields here - usingPortForward: true - config: - peerAs : 65001 - neighborAddress: 192.168.0.6 - addAaths: - sendMax: 10 - transport: - passiveMode: true + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 ``` -1. `neighborAddress` is the IP address of the router. -2. `peerAs` is the AS number of the neighbors, which must be different from the Porter's AS number. Please use private BGP number, the range is 64512 – 65534 -3. `sendMax` specifies the upper limit of the sending route, if you want to enable the ECMP feature, this value must be greater than 1. -4. `usingPortForward` turns on port forwarding and is used for switches that do not support ports other than 179, such as Cisco switches. -5. `passiveMode` indicate porter manager connect router voluntarily. +1. `conf.neighborAddress` is the IP address of the router. +2. `conf.peerAs` is the Autonomous System of the router and must be different from the cluster. If it is a private network, generally use an Autonomous System above 65000. + +### Specify sendMax + +`sendMax` is used to indicate the maximum number of equivalent routes that gobgp can send when sending ECMP routes; the default is 10. It can be specified in the following configuration + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + name: bgppeer-sample +spec: + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 + afiSafis: + - config: + family: + afi: AFI_IP + safi: SAFI_UNICAST + enabled: true + addPaths: + config: + sendMax: 10 +``` + +### Specify nodeSelector + +When BgpPeer is created, by default all replicas of Porter Manager will respond to this configuration and establish a connection with it, but in some scenarios where Kubernetes cluster nodes are deployed under different routers, you need to specify the relationship between gobgp and the router to establish a connection by setting `nodeSelector`. + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + name: bgppeer-sample +spec: + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 + nodeSelector: + matchLabels: + kubernetes.io/hostname: node4 +``` -`porter` only uses a small part of the functions in gobgp. For more details, please refer to [gobgp configuration](https://github.com/osrg/gobgp/blob/master/docs/sources/configuration.md) \ No newline at end of file +The above configuration means that only Porter Manager on node4 will establish a BGP connection with 172.22.0.2. \ No newline at end of file diff --git a/doc/compared_with_metallb.md b/doc/compared_with_metallb.md index b65f1479a..3c20c72cd 100644 --- a/doc/compared_with_metallb.md +++ b/doc/compared_with_metallb.md @@ -1,14 +1,85 @@ -# Compared with MetalLB - -## Pros -- Support most BGP features and multiple network architectures. -- A Kubernetes-friendly tool based on CRD-Controller that can be controlled entirely by kubectl. -- The configuration file can be updated dynamically without any restart. BGP configurations are automatically updated based on the network environment. Various BGP features can be dynamically adopted. -- Provide Passive mode and support DNAT. -- Conflicts with Calico can be handled in a more friendly way. - -## Cons - - Support Linux only. - -## Similarity -- More tests are needed for both tools. \ No newline at end of file +# Compared to MetalLB + +> English | [中文](zh/compared_with_metallb.md) + + +* Cloud Native Architecture + +In Porter, you can use CRD to configure both address management and BGP configuration management, and you can view resource status, which is simple and extensible. + +In MetalLB, they are all configured via configmap, and you have to check monitoring or logging to see their status. + +* Address Management + +In Porter, addresses are managed via the Eip CRD, which defines the child resource status to store the status of the address assignment, so that there is no conflict between copies of the address assignment, and the programming logic is simple. + +* Publishing routes with gobgp + +Unlike MetalLB's implementation of the BGP protocol itself, Porter uses [gobgp](https://github.com/osrg/gobgp/blob/master/docs/sources/lib.md) to distribute routes, which has the following advantages. +1. low development cost and supported by the gobgp community +2. can take advantage of the rich features of gobgp + +* Dynamic Configuration of gobgp via BgpConf/BgpPeer CRD + +When gobgp is used as a lib, the community provides an [API](https://github.com/osrg/gobgp/blob/master/api/gobgp.pb.go) based on the protobuf, which Porter also refers to when implementing the BgpConf/BgpPeer CRD , and maintain compatibility with the gobgp API. + +Porter also provides status for viewing the BGP neighbor configuration, which is rich in status information. + +```bash +root@node1:/tmp# kubectl get bgppeers.network.kubesphere.io bgppeer-sample -o yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"network.kubesphere.io/v1alpha2","kind":"BgpPeer","metadata":{"annotations":{},"name":"bgppeer-sample"},"spec":{"conf":{"neighborAddress":"172.22.0.2","peerAs":50000}}} + creationTimestamp: "2020-11-20T09:00:52Z" + finalizers: + - finalizer.lb.kubesphere.io/v1alpha1 + generation: 5 + name: bgppeer-sample + resourceVersion: "6634958" + selfLink: /apis/network.kubesphere.io/v1alpha2/bgppeers/bgppeer-sample + uid: 70bdd404-b01a-46ec-a7fe-e307a3fa41e8 +spec: + conf: + neighborAddress: 172.22.0.2 + peerAs: 50000 + nodeSelector: + matchLabels: + kubernetes.io/hostname: node4 +status: + nodesPeerStatus: + node4: + peerState: + messages: + received: + keepalive: "170" + open: "1" + total: "173" + update: "2" + sent: + keepalive: "149" + open: "1" + total: "150" + neighborAddress: 172.22.0.2 + peerAs: 50000 + peerType: 1 + queues: {} + routerId: 198.51.100.1 + sessionState: ESTABLISHED + timersState: + downtime: "2020-11-24T04:51:53Z" + keepaliveInterval: "30" + negotiatedHoldTime: "90" + uptime: "2020-11-24T04:51:53Z" +``` + + +* Simple architecture and low resource consumption + +Porter is currently deploying Deployment only, which enables high availability through multiple replicas **Not all replicas are crashed without affecting the normal established connections**. + +In BGP mode, different replicas of the Deployment will connect to the router for issuing equivalent routes, so normally we can deploy two replicas. + +In layer2 mode, the replicas elect a leader via the Leader Election mechanism provided by Kuberenetes, which in turn responds to ARP/NDP. \ No newline at end of file diff --git a/doc/deploy.md b/doc/deploy.md new file mode 100644 index 000000000..ab4f8c893 --- /dev/null +++ b/doc/deploy.md @@ -0,0 +1,48 @@ +# Installation + +> English | [中文](zh/deploy.md) + +## Prerequisites + +* BGP mode + +1. The router must support the BGP protocol. +2. Requires the router to support ECMP and includes the following features. + - Support for receiving multiple equivalence routes + - Supports receiving multiple equivalent routes from the same neighbor. + +* layer2 mode + +In layer2 mode, you need to enable `strictARP`, which disables the network card from answering ARP requests from IP addresses on other network cards. + +```yaml +kubectl edit configmap -n kube-system kube-proxy + +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +mode: "ipvs" +ipvs: + strictARP: true +``` + +Then restart the kube-proxy + +```bash +kubectl rollout restart -n kube-system daemonset kube-proxy +``` + +## Installation via kubectl + +To install Porter in one click, run the following command + +```bash +kubectl apply -f https://raw.githubusercontent.com/kubesphere/porter/master/deploy/porter.yaml +``` + +## Installation via chart package + +```bash +helm repo add test https://charts.kubesphere.io/test +helm repo update +helm install porter test/porter +``` \ No newline at end of file diff --git a/doc/deploy_baremetal.md b/doc/deploy_baremetal.md deleted file mode 100644 index e34f6c7c7..000000000 --- a/doc/deploy_baremetal.md +++ /dev/null @@ -1,143 +0,0 @@ -# Deploy Porter on Bare Metal Kubernetes Cluster - -> English | [中文](zh/deploy_baremetal.md) - -## Prerequisites -1. A Kubernetes cluster -1. Your router needs to support the BGP protocol -1. Your router needs to support Equal-cost multi-path routing (ECMP) if you want to enable load-balancing on the router. Including the following features: - - Support multi-path routing - - Support BGP Additional-Paths -1. If there is a router that does not support the BGP protocol (or is not allowed to enable the BGP protocol), you need to manually write the nexthop route of EIP on this router (or use other routing protocols) - -## Install Porter -1. Install Kubernetes Cluster -2. Get Porter's YAML file - ```bash - wget https://github.com/kubesphere/porter/releases/download/v0.1.1/porter.yaml - ``` -3. You need to modify a `ConfigMap` named `bgp-cfg` in the YAML according to the [BGP Configuration](bgp_config.md) -4. Install Porter on k8s cluster - ```bash - kubectl apply -f porter.yaml - ``` - -## Router Configuration -> Different routers have different configurations. Here is the configuration of a Cisco Nexus 9000 Series. For more router configuration, please refer to [Router Configuration](router_config.md). - -### [Cisco Nexus 9000 Series](https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/92x/unicast/configuration/guide/b-cisco-nexus-9000-series-nx-os-unicast-routing-configuration-guide-92x/b-cisco-nexus-9000-series-nx-os-unicast-routing-configuration-guide-92x_chapter_01010.html) - - -1. Enter the N9K configuration interface as admin. Modify the following configuration according to the actual situation. - - ``` - feature bgp - - router bgp 65001 - router-id 10.10.12.1 - address-family ipv4 unicast - maximum-paths 8 - additional-paths send - additional-paths receive - neighbor 10.10.12.5 - remote-as 65000 - timers 10 30 - address-family ipv4 unicast - route-map allow in - route-map allow out - soft-reconfiguration inbound always - capability additional-paths receive - ``` - -2. After the configuration is complete, check the neighbor status as `Established`. - - ```bash - myswitvh(config)# show bgp ipv4 unicast neighbors - - BGP neighbor is 10.10.12.5, remote AS 65000, ebgp link, Peer index 3 - BGP version 4, remote router ID 10.10.12.5 - BGP state = Established, up for 00:00:02 - Peer is directly attached, interface Ethernet1/1 - Last read 00:00:01, hold time = 30, keepalive interval is 10 seconds - Last written 0.996717, keepalive timer expiry due 00:00:09 - Received 5 messages, 0 notifications, 0 bytes in queue - Sent 13 messages, 0 notifications, 0(0) bytes in queue - Connections established 1, dropped 0 - Last reset by us 00:01:29, due to session closed - Last reset by peer never, due to No error - - Neighbor capabilities: - Dynamic capability: advertised (mp, refresh, gr) - Dynamic capability (old): advertised - Route refresh capability (new): advertised received - Route refresh capability (old): advertised - 4-Byte AS capability: advertised received - Address family IPv4 Unicast: advertised received - Graceful Restart capability: advertised - ``` - -## Deployment -1. Add an EIP pool - - ```bash - kubectl apply -f - <` . (recommended) - - Add `eip.porter.kubesphere.io/v1alpha1: ` to `annotations`. - - - - -3. On the router we can see that a new network (external IP address) was added with three paths. Each path is linked to one of the nodes: - - ``` - # show bgp all - - 10.11.11.11/32, ubest/mbest: 3/0 - *via 10.10.12.2, [20/0], 00:03:38, bgp-65001, external, tag 65000 - *via 10.10.12.3, [20/0], 00:03:38, bgp-65001, external, tag 65000 - *via 10.10.12.4, [20/0], 00:03:38, bgp-65001, external, tag 65000 - - ``` -4. Use `kubectl get eip` to watch the current usage of EIP \ No newline at end of file diff --git a/doc/eip_config.md b/doc/eip_config.md index ae7c5e5b9..df59e98aa 100644 --- a/doc/eip_config.md +++ b/doc/eip_config.md @@ -1,21 +1,90 @@ -#EIP Configuration +# Eip Configuration -Porter only supports ipv4 address now. Let's take this as an example +> English | [中文](zh/eip_config.md) + + +Eip is used to configure IP address segments, which Porter will assign to the LoadBalancer Service, and then publish routes via `BGP/ARP/NDP` protocols. + +**Note: Porter currently only supports IPv4 addresses, support for IPv6 will be completed soon.** + +Eip's example below shows all available configuration fields and a description of the status fields. ```yaml -apiVersion: network.kubesphere.io/v1alpha1 +apiVersion: network.kubesphere.io/v1alpha2 kind: Eip metadata: name: eip-sample-pool spec: address: 192.168.0.0/24 protocol: layer2 + interface: eth0 + disable: false +status: + occupied: false + usage: 1 + poolSize: 256 + used: + "192.168.0.1": "default/test-svc" + firstIP: 192.168.0.0 + lastIP: 192.168.0.255 + ready: true + v4: true ``` -1. The protocol should be layer 2 or BGP. -2. The address supports three kinds of syntax. +## spec field explanation + +* address + +`address` is used to describe a range of IP addresses, which can have the following three formats + ```yaml - ip e.g. 192.168.0.1 - ip/net e.g. 192.168.0.0/24 - ip1-ip2 e.g. 192.168.0.1-192.168.0.10 -``` \ No newline at end of file +``` + +**Note: The IP address segment must not overlap with other created Eip, otherwise the resource creation error will occur.** + +* protocol + +`protocol` is used to describe what protocol is used to publish routes, and the valid values are `layer2` and `bgp`. When the value is null, the mode protocol is `bgp`. + +* interface + +`interface` makes sense when `protocol` is `layer2` and is used to indicate which network card Porter is listening for ARP/NDP requests on. + +When the NIC names in each node of a Kubernetes cluster are different, you can specify the NIC by using the syntax `interface: can_reach:192.168.1.1`. In the above example, Porter gets the first NIC in the route by finding the route to 192.168.1.1. + +* disable + +With `true`, Porter will not be assigned an address from this Eip when a new LoadBalancer Service is created, but it will not affect a Service already created. + +## status field explanation + +* occupied + +This field is used to indicate whether or not an address in Eip has been allocated and used up. + +* usage 和 used + +`usage` is used to indicate how many addresses have been allocated in Eip; `used` is used to indicate which address is being used by which Service, key is the IP address, value is the Service's `Namespace/Name`. + +* poolSize + +This field is used to indicate the total number of addresses in Eip. + +* firstIP + +This field is used to represent the first IP address in Eip. + +* lastIP + +This field is used to represent the last IP address in Eip. + +* v4 + +This field is used to represent the address protocol family of Eip. + +* ready + +This field is used to indicate whether the BGP/ARP/NDP related program associated with Eip has been initialized or not. \ No newline at end of file diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 000000000..04f04a0e0 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,18 @@ +## Install Porter + +* [deploy](./deploy.md) + +## Configure Porter + +* [bgp_config](./bgp_config.md) +* [eip_config](./eip_config.md) + +## Use Porter + +* [usage](./usage.md) +* [layer2](./layer2.md) +* [simulate_with_bird](./simulate_with_bird.md) + +## Porter VS. MetalLB + +* [compared_with_metallb](./compared_with_metallb.md) \ No newline at end of file diff --git a/doc/layer2.md b/doc/layer2.md index 123ff3e8f..7fff230f9 100644 --- a/doc/layer2.md +++ b/doc/layer2.md @@ -1,53 +1,120 @@ -#Preparation +# Porter layer2 mode -Normally, Nic will answer ARP request which ip not reside on it. In order to let porter-manager -control all arp reply for layer2 eip, we should config kube-proxy strictARP. -```yaml -kubectl edit configmap -n kube-system kube-proxy +> English | [中文](zh/layer2.md) -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -mode: "ipvs" -ipvs: - strictARP: true -``` -# Porter Layer2 Usage -Porter now support bgp and layer2, and will support non-bgp switch in future. In -order to distinguish them, filed `protocol` was added. +Most routers now support BGP, but in practice it may be inconvenient to open up BGP for some reason, such as security compliance, or if the physical router you are using is too old to support BGP, then you can configure Porter layer2 mode to achieve similar functionality. + +## Layer2 Principle + +When a client accesses the server via IP, since the configured Eip is on the same Layer 2 network as the Kubernetes cluster, the router will send an ARP/NDP request to find the MAC address of the Eip. At this time Porter will answer the MAC of the Kubernetes Node according to the Endpoints of the LoadBalancer Service. After ARP/NDP is answered, subsequent client traffic is sent to the same Node. + +Due to the one-to-one correspondence between IP and MAC, the LoadBalancer Service can only answer the MAC address of the same Node during its lifetime, unless Endpoints change. To achieve this, Porter uses Kubernetes' own Leader Election feature, which allows only one copy to answer ARP/NDP requests. + +**Limitations: There is a single point of failure when the client connects to the server via Eip, and all traffic from Eip is sent to the same Node**. + +Translated with www.DeepL.com/Translator (free version) -An example of how to use it will follow. -Create a layer2 EIP +## The use of layer2 + +The layer2 mode is much simpler to use than the BGP mode. You just need to configure the layer2 mode Eip, and specify the protocol as layer2 when creating the workload. + +* Create layer2 Eip ```yaml kubectl apply -f - < Ubuntu 18.04.4 LTS 4.15.0-109-generic docker://19.3.6 +node3 Ready worker 18d v1.17.9 172.22.0.9 Ubuntu 18.04.4 LTS 4.15.0-108-generic docker://19.3.8 +node4 Ready worker 18d v1.17.9 172.22.0.10 Ubuntu 18.04.4 LTS 4.15.0-101-generic docker://19.3.8 +root@node1:~# +``` + +The above operation reveals that the next hop for 172.22.0.188 is 172.22.0.3, since they both have the same MAC address and point to the same node, node1. + diff --git a/doc/porter-chart.md b/doc/porter-chart.md deleted file mode 100644 index a4147a720..000000000 --- a/doc/porter-chart.md +++ /dev/null @@ -1,298 +0,0 @@ -# Porter Chart - -> English | [中文](zh/porter-chart.md) - -# Install Porter using Helm Chart - -```bash -helm repo add test https://charts.kubesphere.io/test -helm repo update -helm install porter test/porter -``` - -# Layer 2 mode - -## Prerequistes - -- Requires Kubernetes `1.17.3` or above - -- A linux machine, used to detect LoadBalancer of nginx - -## Configure layer2 in kubernetes - -```bash -$ cat << EOF > layer2.yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: Eip -metadata: - name: eip-sample-pool -spec: - # Modify the ip address segment to the ip address segment of the actual environment. It can be a single address or an address segment - address: 192.168.3.100 - protocol: layer2 - disable: false -EOF -$ kubectl apply -f layer2.yaml -eip.network.kubesphere.io/eip-sample-pool created -``` - -## Deploy Nginx App - -Execute commands on Kubernetes cluster: - -```bash -$ cat << EOF > nginx-layer2.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - lb.kubesphere.io/v1alpha1: porter - protocol.porter.kubesphere.io/v1alpha1: layer2 - name: nginx-service -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 8088 - targetPort: 80 -EOF -$ kubectl apply -f nginx-layer2.yaml -deployment.apps/nginx-deployment created -service/nginx-service created -$ kubectl get svc/nginx-service -default kubernetes ClusterIP 10.96.0.1 443/TCP 129m -default nginx-service LoadBalancer 10.100.5.90 192.168.3.100 8088:32063/TCP 50s -``` - -## Access Nginx Service - -Execute commands on linux: - -```bash -$ curl 192.168.3.100:8088 -``` - -# BGP mode - -## Prerequistes - -- Requires Kubernetes `1.17.3` or above. - -- Require `Router` to start BGP mode.We will install bird on Centos7 system and use bird to implement BGP routing function. We call this machine `Router`. - -- A linux machine, used to detect LoadBalancer of nginx - -## Network diagram - -```bash - ________________ ________________ ________________ -| | | | | | -| k8s cluster | <--------- | Router | <--------- | other host | -|_______________| |________________| |________________| -``` - -- We use bird to implement BGP mode on centos7 system. - -- Other hosts send packets to a `Router`, and the `Router` is sending packets to the k8s cluster. - -- The k8s cluster needs to use BGP to establish a connection with the `Router`, so the `as` of the two must be different. - -## Configure on Router - -Install bird on `Router`: - -```bash -$ yum install bird -$ systemctl enable bird -``` - -Configure BGP on the `Router` as follows: - -```bash -cat /etc/bird.conf -protocol kernel { - scan time 60; # Scan kernel routing table every 20 seconds - import none; # Default is import all - export all; # Default is export none - merge paths on; # Enable ECMP, this parameter requires at least bird 1.6 -} - -protocol device { - scan time 10; # Scan interfaces every 10 seconds -} - -protocol static { -} - -protocol bgp mymaster { - description "10.55.0.127"; # local ip - local as 65001; # local as.It must be different from the as of the port-manager - neighbor 10.55.0.124 port 17900 as 65000; # Master node IP and AS number - source address 10.55.0.127; # Router IP - import all; - export all; - enable route refresh off; - add paths on; -} -``` - -Start bird on the `Router` and set ipv4 forwarding: - -```bash -$ systemctl restart bird -$ sysctl -w net.ipv4.ip_forward=1 -``` - -Check whether the configuration takes effect on the `Router`, you will see `mymaster` rule. - -```bash -$ birdc show protocol -BIRD 1.6.8 ready. -name proto table state since info -kernel1 Kernel master up 18:01:55 -device1 Device master up 18:01:55 -static1 Static master up 18:01:55 -mymaster BGP master start 18:01:55 Active Socket: Connection refused -``` - - -## Establish BGP connection on porter and Router - -Execute commands on Kubernetes cluster: - -```bash -$ cat << EOF > bgp.yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: Eip -metadata: - name: eip-sample-pool -spec: - # Modify the ip address segment to the ip address segment of the actual environment. - address: 10.55.0.100 - protocol: bgp - disable: false ---- -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpConf -metadata: - name: bgpconf-sample -spec: - # the as of porter - as : 65000 - routerID : 10.55.0.124 - port: 17900 ---- -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpPeer -metadata: - name: bgppeer-sample -spec: - # the as of the Router - config: - peerAs : 65001 - neighborAddress: 10.55.0.127 - addPaths: - sendMax: 10 -EOF -$ kubectl apply -f bgp.yaml -eip.network.kubesphere.io/eip-sample-pool created -bgpconf.network.kubesphere.io/bgpconf-sample created -bgppeer.network.kubesphere.io/bgppeer-sample created -``` - -Check whether the connection is established on the `Router`, and the info information shows `Established` means the connection is established. - -```bash -$ birdc show protocol -BIRD 1.6.8 ready. -name proto table state since info -kernel1 Kernel master up 18:10:39 -device1 Device master up 18:10:39 -static1 Static master up 18:10:39 -mymaster BGP master up 18:15:45 Established -``` - -## Deploy Nginx App - -```bash -$ cat << EOF > nginx-bgp.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - lb.kubesphere.io/v1alpha1: porter - protocol.porter.kubesphere.io/v1alpha1: bgp - name: nginx-service -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 8088 - targetPort: 80 -EOF -$ kubectl apply -f nginx-bgp.yaml -deployment.apps/nginx-deployment created -service/nginx-service created -``` - -## Access Nginx Service - -If other machines in the LAN want to access nginx, you need to set up routing. Forward the packet to the `Router`. - -```bash -$ # "-host" refers to a single machine, if you need to specify a network segment, please use "-net" -$ #"192.168.3.100" refers to the address of the application service.This use the nginx service address -$ #"192.168.3.85" refers to the Router address. -$ route add -host 192.168.3.100 gw 192.168.3.85 eth0 -``` - -```bash -$ curl 192.168.3.100:8088 -``` - diff --git a/doc/router_config.md b/doc/router_config.md deleted file mode 100644 index d82f41d10..000000000 --- a/doc/router_config.md +++ /dev/null @@ -1,3 +0,0 @@ -# Router Configuration - -> English | [中文](zh/router_config.md) diff --git a/doc/simulate_with_bird.md b/doc/simulate_with_bird.md index 7e726892f..243a7e5a8 100644 --- a/doc/simulate_with_bird.md +++ b/doc/simulate_with_bird.md @@ -1,177 +1,194 @@ -# Simulate with Bird +# Simulate with bird > English | [中文](zh/simulate_with_bird.md) -> This article is an experiment done on [QingCloud](https://www.qingcloud.com/) platform. There are some differences in configuration for different platforms. The advantage of simulation is that you can experience the function of Porter without touching the actual hardware, but it is still different from the real router. The simulated router has a default network card (not used for routing), which will cause the default route when the packet goes back. In addition, when configuring the simulated router, there are many additional parameters, please set these parameters according to this article. +As an ordinary developer, it is difficult to come across hardware routers in your daily work. Fortunately, there are many software programs that provide similar functionality to physical routers, such as [bird](https://bird.network.cz/) and [gobgp](https://osrg.github.io/gobgp/), which can also assist with We develop. ## Prerequisites -1. A k8s cluster -2. Ensure that the host used to simulate the router can be connected to the k8s cluster, including the bgp port and the application port in the cluster. -3. A public IP address - - -## Create a router - -1. Create a host in the k8s network and install Bird on the host. The QingCloud platform only has Bird 1.5 version. This version does not support ECMP. To experience all the functions of Porter, you need to install at least 1.6 version. Execute the following script to install Bird 1.6. - ``` - $sudo add-apt-repository ppa:cz.nic-labs/bird - $sudo apt-get update - $sudo apt-get install bird - $sudo systemctl enable bird - ``` - -2. Configure the router's BGP service. Modify `/etc/bird/bird.conf` and add the following parameters: - ``` - protocol bgp mymaster { - description "192.168.1.4"; # Router ID, usually the main IP address - local as 65001; # Local AS number, must be different from the AS number of the k8s cluster - neighbor 192.168.1.5 port 17900 as 65000; # Master node IP and AS number - source address 192.168.1.4; # Router IP - import all; - export all; - enable route refresh off; # Due to the low BGP protocol of bird 1.6, multiple routes advertised by Porter will become a single route, this parameter can be used as a workaround to fix this problem. - add paths on; # When this parameter is set to on, you can receive multiple routes from the Porter. - } - ``` - The above parameters configure a neighbor to the simulated router. The neighbor is the master node of the cluster. **We assume that your Porter controller is deployed on the master node. If you do not want to restrict the porter to be deployed on the master node or the master cannot deploy pods, then you need to add all neighbor nodes to this configuration file according to the above rules.** Modify the `kernel` part of the file, cancel the `export all` comments, and enable the ECMP function: - ``` - protocol kernel { - scan time 60; - import none; - export all; # Actually insert routes into the kernel routing table - merge paths on; # Enable ECMP, this parameter requires at least bird 1.6 - } - - ``` - -3. Reboot Bird - ```bash - $sudo systemctl restart bird - ``` - -4. Configure the Elastic IP. Apply an Elastic IP that **Associate Mode is Internal** on the QingCloud console. Please note that EIP must be an internal associate mode. If it is an external associate mode, EIP cannot be found in VM. Associate this EIP to the VM where the simulated router is located. You only need to complete the associate. There is no need to follow up with the QingCloud documentation. - -5. A new network card (usually eth1) will be created when the IP is bound to the QingCloud host. Run `ip a` on the host to check whether the network card is `UP`. If not, execute `ip link set up eth1`. - -6. Turn on port forwarding and turn off packet filtering on the simulated router - ``` - sysctl -w net.ipv4.ip_forward=1 - sysctl -w net.ipv4.conf.all.rp_filter=0 - sysctl -w net.ipv4.conf.eth1.rp_filter=0 - sysctl -w net.ipv4.conf.eth0.rp_filter=0 - ``` -7. Configure the firewall. Do some port tests on QingCloud console. - -8. Configure routing rules. Since the default network card of the simulated router is `eth0`, after the cluster returns the ip packet, it will be sent from `eth0` by default. The user accesses the public network IP from `eth1`, which will cause the transmission to fail. Therefore, the packets sent from the bound IP need to be routed to `eth1`. Access this address on the browser, and use `tcpdump -i eth1` to capture packets on the simulated router, and observe the address of the upper router, such as: - - ```bash - root@i-7bwamgny:~# tcpdump -i eth1 - tcpdump: verbose output suppressed, use -v or -vv for full protocol decode - listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes - 14:24:07.401555 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475097 ecr 0], length 0 - 14:24:07.403573 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475100 ecr 0], length 0 - 14:24:07.654341 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475350 ecr 0], length 0 - 14:24:10.400770 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478097 ecr 0], length 0 - 14:24:10.404100 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478100 ecr 0], length 0 - 14:24:10.658557 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478351 ecr 0], length 0 - 14:24:16.401591 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484098 ecr 0], length 0 - 14:24:16.404605 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484101 ecr 0], length 0 - 14:24:16.656750 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484351 ecr 0], length 0 - - ``` - In the above output, `139.198.121.228` is the bound IP, and the left side of the `>` is the address of the upper router. Configure the rules for returning packets through routing policies: - ```bash - sudo ip rule add from 139.198.254.4/32 lookup 101 # If the packet comes from this IP, then go to the routing table 101 - sudo ip route replace default dev eth1 table 101 # Set the default network card of routing table 101 to eth1 - ``` - The actual physical router does not need to configure the above rules, because the router knows how to configure this IP correctly. **If you need to access and test ECMP from multiple IP addresses, then these IPs also need to be configured with the same steps** + +1. have a functioning Kubernetes cluster +2. Porter is correctly installed +3. hosts with bird installed interoperate with the Kubernetes node network. + +## Create and configure bird + +1. Install bird -9. After the configuration is completed, you can execute `birdc show protocol` to view the connection information. - -> Note: If the way to connect to the host is through the public IP, then after performing the above operation, it is possible that the SSH connection will be disconnected (if and only if the public IP of SSH and the public IP of your test are in QingCloud The network will be NAT to the above 139.198.254.4/32). After disconnection, you can use the VNC to connect the host on the QingCloud website. It is recommended to use VPN connection. The following operations in the k8s cluster will have the same effect. - -## Configure plugins -> All the operations are in the master node of the k8s cluster - -1. Get Porter's YAML file - ``` - wget https://github.com/kubesphere/porter/releases/download/v0.1.1/porter.yaml - ``` -2. You need to modify a `ConfigMap` named `bgp-cfg` in the YAML according to the [BGP Configuration](bgp_config.md) -3. Configure public network IP routing rules. Same as the problem with the simulated router. **This step needs to be configured on all k8s nodes, because the actual service may be deployed on any node.** - ```bash - sudo ip rule add to 139.198.254.0/24 lookup 101 - sudo ip route replace default via 192.168.98.5 dev eth0 table 101 # 192.168.98.5 is the router IP - ``` - The above `192.168.98.5` is the address of the simulated router. A loop rule has been configured on the simulated router, so this packet will not be dropped. The actual k8s cluster does not need to be configured, because the default gateway of the k8s cluster is this router. - -4. Install Porter on k8s cluster: `kubectl apply -f porter.yaml` -5. Add and EIP - ```bash - kubectl apply -f - <` . (recommended) - - Add `eip.porter.kubesphere.io/v1alpha1: ` to `annotations`. - -7. Check the Porter logs and EIP events, if there is no problem, you can access the service according to the EIP. - ```bash - kubectl logs -n porter-system controller-manager-0 -c manager # Check the logs of Porter - kubectl describe eip eip-sample # Check the events - ``` -8. Check if there are two equal-cost routes on the simulated router: - ```bash - root@i-7bwamgny:~# ip route - default via 192.168.98.1 dev eth0 - 139.198.121.228 proto bird - nexthop via 192.168.98.2 dev eth0 weight 1 - nexthop via 192.168.98.4 dev eth0 weight 1 - ``` - -## Test the load balancing of ECMP -> Note: The kernel version of the host where the simulated router is located must be higher than 3.6. The default kernel version of QingCloud platform is 4.4. The ECMP Hash algorithm used is `L3`. ECMP will only adjust the access route based on the source IP. The kernel versions above 4.12 support `L4`. You can run `sysctl net.ipv4.fib_multipath_hash_policy 1` to change the load balancing hash algorithm, then use `curl` to access this eip, so that you can achieve the effect of load balancing. - -The actual router only needs to enable the ECMP function to achieve load balancing. In order to test the effectiveness of load balancing, you need to access this EIP from different source IPs and observe whether there is traffic on each node. - -1. Observe the node where the Pod is located +`1.5 does not support ECMP, you need at least 1.6 to experience the full functionality of Porter`, in ubuntu you can install bird 1.6 by executing the following script + +``` +$sudo add-apt-repository ppa:cz.nic-labs/bird ##这里引入了bird 1.6 +$sudo apt-get update +$sudo apt-get install bird +$sudo systemctl enable bird +``` + +2. Configuring the bird + +The configuration file of bird is `/etc/bird/bird.conf`, please refer to [bird official documentation](https://bird.network.cz/?get_doc&f=bird.html&v=16). + +* Configuring the router id + +The format of `router id` is a valid IP address, please change it according to your environment. + +``` +router id 172.22.0.2; +``` + +* Configuring the bgp neighbor + +Configure the bgp neighbor according to the actual deployment node of `Porter Manager`, **If you have multiple nodes, please add multiple neighbors**. + +``` +protocol bgp neighbor1 { + local as 65001; #填本地AS域，必须和Kubernetes集群的AS不同 + neighbor 172.22.03 port 17900 as 65000; ##填master节点IP和 AS域 + source address 172.22.0.2; #填本交换机IP + import all; + export all; + enable route refresh off; #由于bird1.6的bgp较低，和Porter的bgp连接会将多路由变成单个路由，这个参数能够作为一个workaround修正这个问题。 + add paths on; #这个参数开启之后，就可以收到porter发来的多个路由并同时存在而不会覆盖。 +} +``` + +* Configuring the bird kernel + +Add `export all;` and `merge paths on;` for adding routes to the linux kernel. + +``` +protocol kernel { + scan time 60; + import none; + export all; + merge paths on; +} +``` + +3. Restart bird + ```bash -kubectl get pod -o wide +$sudo systemctl restart bird ``` -2. Observe whether each node has set routing rules + +4. Confirm Bird + +Check to see if the bird configuration is starting properly by running the following command. If the status is not `active`, you can check for errors by running `journalctl -f -u bird`. + ```bash -root@master-k8s:~# ip rule -0: from all lookup local -32763: from all to 139.198.121.228 lookup 101 +$sudo systemctl status bird ``` -3. Run `tcpdump -i eth0 port $port` on these nodes, where `$port` is the port exposed by the service. In the above example, it is 8088. -4. Access this EIP from different IPs and observe whether there is traffic on these nodes. If there is any, then there is no problem with load balancing. +## Configuring Porter + +* Configuring BgpConf + +Please refer to [bgp_config] (. /bgp_config.md), and modify the following configuration as you see fit. + +```yaml +kubectl apply -f - < English | [中文](zh/usage.md) + + +After Porter is installed and configured, you can use Porter exposure by creating a LoadBalancer Service. + +## Configuring LoadBalancer Service + +The LoadBalancer Service must add annotations `lb.kubesphere.io/v1alpha1: porter` and the type must be specified as `LoadBalancer`. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +**Porter does not handle other non-qualified Services. ** + +### Specify Protocol + +You can specify the protocol by which Porter assigns addresses by specifying annotation "protocol.porter.kubesphere.io/v1alpha1". + +The following example specifies annotation "protocol.porter.kubesphere.io/v1alpha1: bgp", which means that Porter will allocate the address from the BGP protocol's Eip. It is also the default value for Porter and can be omitted. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +The following example specifies annotation "protocol.porter.kubesphere.io/v1alpha1: layer2", which means that Porter will allocate the address from the layer2 protocol's Eip. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: layer2 +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +### Specify Eip + +Suppose we have the following example of Eip + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: Eip +metadata: + name: eip-sample-pool +spec: + address: 192.168.0.0/24 +``` + +Porter iterates through all `enable` Eip's when processing the LoadBalancer Service, **allocating addresses from the first Eip that matches the protocol and has a free address**, which means that the allocated address will be unpredictable. Service specifies the parameter `spec.loadBalancerIP` to fix the address. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp +spec: + selector: + app: mylbapp + type: LoadBalancer + loadBalancerIP: 192.168.0.100 + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +If you don't need a fixed address, but just need to allocate addresses from a specific address pool, then you can specify annotation "eip.porter.kubesphere.io/v1alpha2", which takes the value of Eip's Name. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp + eip.porter.kubesphere.io/v1alpha2: eip-sample-pool +spec: + selector: + app: mylbapp + type: LoadBalancer + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +### Configure spec.externalTrafficPolicy + +By default, the LoadBalancer Service's `externalTrafficPolicy` value is `Cluster`, which means that all nodes in a Kubernetes cluster can be used as Nexthop to forward traffic, the difference is simply that. +* In BGP mode, Porter publishes equivalent routes containing all Nodes as Nexthop. +* Under layer2, Porter randomly selects a Node as a Nexthop to forward traffic. + +Kubernetes also provides an additional value of `Local` for `externalTrafficPolicy`, which means that only Nodes contained in Endpoints in a Kubernetes cluster can be used as Nexthop to forward traffic. + +### Share Eip + +Generally, the number of exposed public IP addresses is limited, and when there are many services to be exposed, the Eip address will not be enough, **Porter currently supports BGP mode Eip sharing**. To use it, you just need to specify the Eip you want to share as described in the **Specify Eip** section above. + +**Note: The two Service ports ports cannot be duplicated, and need externalTrafficPolicy not to be Local**. + +The following is an existing Service and specifies `loadBalancerIP: 192.168.0.100`. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp +spec: + selector: + app: mylbapp + type: LoadBalancer + loadBalancerIP: 192.168.0.100 + ports: + - name: http + port: 8088 + targetPort: 80 +``` + +Here, if you want to multiplex the 192.168.0.100 IP, you can just create a Service and specify the IP at the same time. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: mylbapp-svc2 + annotations: + lb.kubesphere.io/v1alpha1: porter + protocol.porter.kubesphere.io/v1alpha1: bgp +spec: + selector: + app: mylbapp + type: LoadBalancer + loadBalancerIP: 192.168.0.100 + ports: + - name: http + port: 8089 + targetPort: 80 +``` \ No newline at end of file diff --git a/doc/usecases.md b/doc/usecases.md deleted file mode 100644 index 56d9d706a..000000000 --- a/doc/usecases.md +++ /dev/null @@ -1,188 +0,0 @@ -# Layer2 - -[Layer2 guide](layer2.md) demonstrates how to install Porter and use Layer 2 on Kubernetes. - -## Prerequisite - -The cluster should have at least two nodes, nodeA and nodeB. - -## Test endpoint update - -- Create a Service test-svc which has one endpoint on nodeA. -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: mylbapp - labels: - app: mylbapp -spec: - containers: - - name: helloworld - image: karthequian/helloworld - imagePullPolicy: IfNotPresent - nodeSelector: - kubernetes.io/hostname: nodeA ---- -kind: Service -apiVersion: v1 -metadata: - name: test-svc - annotations: - lb.kubesphere.io/v1alpha1: porter - #eip.porter.kubesphere.io/v1alpha1: 1.1.1.1 - #protocol.porter.kubesphere.io/v1alpha1: layer2 -spec: - selector: - app: mylbapp - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 -``` - -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip neigh show to ${eip} dev ${nic}', we can find lladdr is nodeA's Mac -- Delete the endpoint which on nodeA, and create endpoint on nodeB -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: mylbapp - labels: - app: mylbapp -spec: - containers: - - name: helloworld - image: karthequian/helloworld - imagePullPolicy: IfNotPresent - nodeSelector: - kubernetes.io/hostname: nodeB -``` -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip neigh show to ${eip} dev ${nic}', we can find lladdr is nodeB's Mac - -# BGP - -[BGP guide](simulate_with_bird.md) demonstrates how to install Porter and use BGP on Kubernetes. - -## Prerequisite - -The cluster should have at least two nodes, nodeA and nodeB. - -## Test BGP PortForward -- Configure BgpConf with port 17900 -```yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpConf -metadata: - name: bgpconf-sample -spec: - # Add fields here - as : 65000 - routerID : 192.168.0.2 - port: 17900 -``` -- Configure BgpPeer with usingPortForward -```yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpPeer -metadata: - name: bgppeer-sample -spec: - # Add fields here - usingPortForward: true - config: - peerAs : 65001 - neighborAddress: 192.168.0.6 - addAaths: - sendMax: 10 -``` -- Configure bird neighbour with port 179 -- Execute 'birdctl show protocol', the nighbour's state should be up - -## Test BGP passive mode -- Configure BgpConf with port 17900 -- Set passiveMode to true in BgpPeer configuration. -```yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpPeer -metadata: - name: bgppeer-sample -spec: - config: - peerAs : 65001 - neighborAddress: 192.168.0.6 - addAaths: - sendMax: 10 - transport: - passiveMode: true -``` -- Configure bird neighbour with port 17900 -- Execute 'birdctl show protocol', the nighbour's state should be up - -## Test endpoint update -- Create a Service test-svc which has two endpoints on nodeA and nodeB respectively. -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: mylbapp - labels: - app: mylbapp -spec: - containers: - - name: helloworld - image: karthequian/helloworld - imagePullPolicy: IfNotPresent - nodeSelector: - kubernetes.io/hostname: nodeA ---- -apiVersion: v1 -kind: Pod -metadata: - name: mylbapp2 - labels: - app: mylbapp -spec: - containers: - - name: helloworld - image: karthequian/helloworld - imagePullPolicy: IfNotPresent - nodeSelector: - kubernetes.io/hostname: nodeB ---- -kind: Service -apiVersion: v1 -metadata: - name: test-svc - annotations: - lb.kubesphere.io/v1alpha1: porter - #eip.porter.kubesphere.io/v1alpha1: 1.1.1.1 - #protocol.porter.kubesphere.io/v1alpha1: layer2 -spec: - selector: - app: mylbapp - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 80 -``` -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip route get ${eip}' on test node, we can find two routes, one via nodeA and one via nodeB. -- Delete the endpoint which on nodeA -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip route get ${eip}' on test node, we can find one route via nodeA. - -## Test BGP graceful down -- Create a Service test-svc which has one endpoint on nodeA -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip route get ${eip}' on test node, we can find one route via nodeA. -- Execute 'kubectl scale -n porter-system deployment porter-manager --replicas=0' -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip route get ${eip}' on test node, we can find one route via nodeA. -- Execute 'kubectl scale -n porter-system deployment porter-manager --replicas=1' -- Execute 'wget ${eip}' can successfully access to test-svc -- Execute 'ip route get ${eip}' on test node, we can find one route via nodeA. diff --git a/doc/zh/bgp_config.md b/doc/zh/bgp_config.md index 948ce4a18..79238ffb1 100644 --- a/doc/zh/bgp_config.md +++ b/doc/zh/bgp_config.md @@ -2,48 +2,109 @@ > [English](../bgp_config.md) | 中文 -Porter使用了[gobgp](https://github.com/osrg/gobgp)来与外部路由器做路由信息交换，目前用到的参数不多，下面简单介绍如何配置插件用到的BGP服务端。 +Porter使用了[gobgp](https://github.com/osrg/gobgp)来与外部路由器建立BGP连接进行路由交换。 + +Porter提供BgpConf和BgpPeer两个CRD用于分别配置gobgp。 这两个CRD定义参考的[gobgp的API](https://github.com/osrg/gobgp/blob/master/api/gobgp.pb.go), 具体使用可以参考[GoBGP as a Go Native BGP library](https://github.com/osrg/gobgp/blob/master/docs/sources/lib.md) + +## BgpConf + +BgpConf用于配置gobgp的全局配置， 所以他只会有一个起作用，目前Porter只会识别name为`default`的配置。 + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpConf +metadata: + #The porter only recognizes configurations with default names; + #configurations with other names are ignored. + name: default +spec: + as: 50001 + listenPort: 17900 + #Modify the router id as you see fit, if it is not specified + #then the porter will use the node ip as the router id. + routerId: 172.22.0.10 +``` + +1. `as`是集群所在自治域，必须和相连的路由器所在自治域不同，相同会导致路由无法正确传输。 +2. `routerId` 表示集群的Id，一般取Kubernetes主节点主网卡的IP。如果你不指定，那么Porter会选择所在节点的第一个IP作为routerId。 +3. `listenPort`是gobgp监听的端口，默认是179。由于Calico也使用了BGP，并且占用了179端口，所以这里必须指定另外的端口。 + +### 指定gobgp监听IP地址 + +通过`ListenAddresses`指定gobgp监听的IP地址。 -## 全局配置 ```yaml -apiVersion: network.kubesphere.io/v1alpha1 +apiVersion: network.kubesphere.io/v1alpha2 kind: BgpConf metadata: - name: bgpconf-sample + #The porter only recognizes configurations with default names; + #configurations with other names are ignored. + name: default +spec: + as: 50001 + listenPort: 17900 + #Modify the router id as you see fit, if it is not specified + #then the porter will use the node ip as the router id. + routerId: 172.22.0.10 + ListenAddresses: + - 172.22.0.10 +``` + +## BgpPeer + +BgpPeer用于配置gobgp的neighbor， 它可以存在多个，具体配置根据自己网络环境调整。 + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + name: bgppeer-sample spec: - # Add fields here - as : 65000 - routerID : 192.168.0.2 - port: 17900 + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 ``` -1. `as`是集群所在自治域，必须和相连的路由器所在自治域不同，相同会导致路由无法正确传输，具体原因涉及到`EBGP`和`IBGP`两种协议的不同，这里不多加赘述。 -2. `routeID`表示集群的id，一般取k8s主节点主网卡的ip。 -3. `port`是gobgp监听的端口，默认是179。由于calico也使用了BGP，并且占用了179端口，所以这里必须指定另外的端口。如果集群的路由器不支持非179以外的端口，那么需要在port所在节点开启端口转发，将179映射到非标准端口。 +1. `conf.neighborAddress`是路由器所在IP地址。 +2. `conf.peerAs`是路由器的自治域，必须与集群不同，而且还需要同路由器中配置的参数一致。 如果是私网，一般使用65000以上的自治域。 -## 设置邻居 -> 邻居即集群所在的路由器。可以添加多个邻居，大多数情况下只需配置一个。 +### 指定sendMax + +`sendMax`用于表示gobgp发送ECMP路由时，最大等价路由数是多少， 默认为10。 可以通过以下配置指定 ```yaml -apiVersion: network.kubesphere.io/v1alpha1 +apiVersion: network.kubesphere.io/v1alpha2 kind: BgpPeer metadata: name: bgppeer-sample spec: - # Add fields here - usingPortForward: true - config: - peerAs : 65001 - neighborAddress: 192.168.0.6 - addPaths: - sendMax: 10 - transport: - passiveMode: true + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 + afiSafis: + - config: + family: + afi: AFI_IP + safi: SAFI_UNICAST + enabled: true + addPaths: + config: + sendMax: 10 ``` -1. `neighborAddress`是路由器所在IP地址。 -2. `peerAs`是邻居所在自治域，必须与集群不同，而且还需要同路由器中配置的参数一致。 如果是私网，一般使用65000以上的自治域。 -3. `sendMax`指定发送路由的上限，如果要实现ECMP功能，这个值必须大于1 -4. `usingPortForward`开启端口转发，用于交换机不支持179以外的端口，比 -5. `passiveMode`表示porter manager主动连接对端路由器 +### 指定nodeSelector -`porter`只使用了gobgp中的一小部分功能，如果有更多的需求，可以参考[gobgp 配置](https://github.com/osrg/gobgp/blob/master/docs/sources/configuration.md) \ No newline at end of file +当创建BgpPeer之后， 默认所有的Porter Manager副本都会响应这个配置，并与它建立连接，但是在某些场景下， Kubernetes集群节点部署在不同的路由器下，这个时候需要通过设置`nodeSelector`指定gobgp与路由器之间建立连接的关系 +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + name: bgppeer-sample +spec: + conf: + peerAs: 50000 + neighborAddress: 172.22.0.2 + nodeSelector: + matchLabels: + kubernetes.io/hostname: node4 +``` +以上配置表示node4上的Porter Manager才会与172.22.0.2建立BGP连接 diff --git a/doc/zh/compared_with_metallb.md b/doc/zh/compared_with_metallb.md index 276cc3c1b..855eb4e2d 100644 --- a/doc/zh/compared_with_metallb.md +++ b/doc/zh/compared_with_metallb.md @@ -1,13 +1,82 @@ # 与MetalLB相比 -## 优点 -- 支持BGP协议的绝大部分特性。支持多种网络架构。 -- k8s 友好。基于CRD-Controller模式，使用kubectl 控制porter的一切。 -- 配置文件动态更新，无需重启，自动更新BGP配置。根据网络环境灵活配置BGP，动态启用各种BGP特性。 -- 更友好地处理与Calico的冲突，提供Passive模式和端口转发模式 +> [English](../compared_with_metallb.md) | 中文 -## 缺点 - - 无法跨平台，仅支持linux +* 云原生架构 - ## 共同点 -- 我们都需要更多的测试 \ No newline at end of file +在Porter中， 不管是地址管理，还是BGP配置管理， 你都可以使用CRD来配置， 并且可以查看资源状态， 简单且扩展灵活。 + +在MetalLB中， 他们都是通过configmap来配置， 感知它们的状态都得通过查看监控或者日志。 + +* 地址管理 + +在Porter中， 通过Eip CRD来管理地址， 它定义子资源status来存储地址分配状态， 这样就不会存在分配地址时各副本发生冲突， 编程时逻辑也会简单。 + +* 使用gobgp发布路由 + +不同于MetalLB自己实现BGP协议， Porter采用[gobgp](https://github.com/osrg/gobgp/blob/master/docs/sources/lib.md)来发布路由，这样做的好处如下： +1. 开发成本低，且有gobgp社区支持 +2. 可以利用gobgp丰富特性 + +* 通过BgpConf/BgpPeer CRD动态配置gobgp + +gobgp作为lib使用时， 社区提供了基于protobuf的[API](https://github.com/osrg/gobgp/blob/master/api/gobgp.pb.go)， Porter在实现BgpConf/BgpPeer CRD时也是参照该API，并保持兼容。 + +同时， Porter也提供status用于查看BGP neighbor配置， 状态信息丰富： +```bash +root@node1:/tmp# kubectl get bgppeers.network.kubesphere.io bgppeer-sample -o yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: BgpPeer +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"network.kubesphere.io/v1alpha2","kind":"BgpPeer","metadata":{"annotations":{},"name":"bgppeer-sample"},"spec":{"conf":{"neighborAddress":"172.22.0.2","peerAs":50000}}} + creationTimestamp: "2020-11-20T09:00:52Z" + finalizers: + - finalizer.lb.kubesphere.io/v1alpha1 + generation: 5 + name: bgppeer-sample + resourceVersion: "6634958" + selfLink: /apis/network.kubesphere.io/v1alpha2/bgppeers/bgppeer-sample + uid: 70bdd404-b01a-46ec-a7fe-e307a3fa41e8 +spec: + conf: + neighborAddress: 172.22.0.2 + peerAs: 50000 + nodeSelector: + matchLabels: + kubernetes.io/hostname: node4 +status: + nodesPeerStatus: + node4: + peerState: + messages: + received: + keepalive: "170" + open: "1" + total: "173" + update: "2" + sent: + keepalive: "149" + open: "1" + total: "150" + neighborAddress: 172.22.0.2 + peerAs: 50000 + peerType: 1 + queues: {} + routerId: 198.51.100.1 + sessionState: ESTABLISHED + timersState: + downtime: "2020-11-24T04:51:53Z" + keepaliveInterval: "30" + negotiatedHoldTime: "90" + uptime: "2020-11-24T04:51:53Z" +``` + + +* 架构简单，资源占用少 + +Porter目前只用部署Deployment即可， 通过多副本实现高可用，**非全部副本crash之后并不不会影响正常已建立连接**。 + +BGP模式下， Deployment不同副本都会与路由器建立连接用于发布等价路由， 所以正常情况下我们部署两个副本即可。 +在layer2模式下，不同副本之间通过Kuberenetes 提供的Leader Election机制选举leader， 进而应答ARP/NDP。 diff --git a/doc/zh/deploy.md b/doc/zh/deploy.md new file mode 100644 index 000000000..966ab2286 --- /dev/null +++ b/doc/zh/deploy.md @@ -0,0 +1,47 @@ +# 安装部署 + +> [English](../deploy.md) | 中文 + +## 安装前提 + +* BGP模式 + +1. 路由器必须支持BGP协议 +2. 需要路由器支持ECMP，并包括以下特性： + - 支持接收多个等价路由 + - 支持接收来自同一个邻居的多条等价路由 + +* layer2模式 + +在layer2模式下，需要开启`strictARP`, 禁止网卡应答其他网卡上IP地址地ARP请求 + +```yaml +kubectl edit configmap -n kube-system kube-proxy + +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +mode: "ipvs" +ipvs: + strictARP: true +``` + +然后重启kube-proxy +```bash +kubectl rollout restart -n kube-system daemonset kube-proxy +``` + +## 通过kubectl安装 + +执行以下命令即可一键安装Porter + +```bash +kubectl apply -f https://raw.githubusercontent.com/kubesphere/porter/master/deploy/porter.yaml +``` + +## 通过chart包安装 + +```bash +helm repo add test https://charts.kubesphere.io/test +helm repo update +helm install porter test/porter +``` \ No newline at end of file diff --git a/doc/zh/deploy_baremetal.md b/doc/zh/deploy_baremetal.md deleted file mode 100644 index a07871368..000000000 --- a/doc/zh/deploy_baremetal.md +++ /dev/null @@ -1,138 +0,0 @@ -# 物理机部署 - -> [English](../deploy_baremetal.md) | 中文 - -## 安装前提 -1. 物理机连接的路由器必须支持BGP协议 -2. 如果需要实现路由器端的负载均衡，需要路由器支持ECMP，并包括以下特性： - - 支持接收多个等价路由 - - 支持接收来自同一个邻居的多条等价路由 -3. 如果网络架构中存在一个路由器不支持BGP（或者被禁止开启BGP）,那么需要在这个路由器上手动写EIP的nexthop路由（或者通过其他路由发现协议） - -## 安装Porter - 1. 在机器上安装kubernetes - 2. 获取yaml - ```bash - wget https://github.com/kubesphere/porter/releases/download/v0.1.1/porter.yaml - ``` - 3. 修改yaml中一个名为bgp-cfg的configmap，按照[BGP配置教程](bgp_config.md)简单修改一些字段即可。注意要路由器的地址和AS域 - 4. 安装porter到集群中 - ```bash - kubectl apply -f porter.yaml - ``` - -## 路由器配置 -> 不同的路由器配置不同，这边仅列出一个样例的思科三层交换机的配置，更多的请参考[路由器配置](router_config.md)。 - -### [Cisco Nexus 9000 Series](https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/92x/unicast/configuration/guide/b-cisco-nexus-9000-series-nx-os-unicast-routing-configuration-guide-92x/b-cisco-nexus-9000-series-nx-os-unicast-routing-configuration-guide-92x_chapter_01010.html) - - -1. 以admin进入N9K配置界面。按照实际情况修改下面的配置。（注：实际输入不能有注释） - - ``` - feature bgp ##开启BGP功能 - - router bgp 65001 #设置本路由器AS域 - router-id 10.10.12.1 #设置本路由器IP - address-family ipv4 unicast - maximum-paths 8 #开启ECMP，并且最多接受8个等价路由 - additional-paths send # 能够发送多个等价路由 - additional-paths receive # 能够接受多个等价路由 - neighbor 10.10.12.5 #邻居IP - remote-as 65000 #邻居AS，必须和本机AS不同 - timers 10 30 - address-family ipv4 unicast - route-map allow in #允许导入系统路由表 - route-map allow out #允许导出路由表到系统 - soft-reconfiguration inbound always # 自动更新邻居状态 - capability additional-paths receive # 开启接受该邻居多条等价路由的能力 - ``` - -2. 配置完成之后，查看邻居状态为`Established`即可。`show bgp ipv4 unicast neighbors` - - ```bash - myswitvh(config)# show bgp ipv4 unicast neighbors - - BGP neighbor is 10.10.12.5, remote AS 65000, ebgp link, Peer index 3 - BGP version 4, remote router ID 10.10.12.5 - BGP state = Established, up for 00:00:02 - Peer is directly attached, interface Ethernet1/1 - Last read 00:00:01, hold time = 30, keepalive interval is 10 seconds - Last written 0.996717, keepalive timer expiry due 00:00:09 - Received 5 messages, 0 notifications, 0 bytes in queue - Sent 13 messages, 0 notifications, 0(0) bytes in queue - Connections established 1, dropped 0 - Last reset by us 00:01:29, due to session closed - Last reset by peer never, due to No error - - Neighbor capabilities: - Dynamic capability: advertised (mp, refresh, gr) - Dynamic capability (old): advertised - Route refresh capability (new): advertised received - Route refresh capability (old): advertised - 4-Byte AS capability: advertised received - Address family IPv4 Unicast: advertised received - Graceful Restart capability: advertised - ``` - -## 部署示例 -1. 添加一个 EIP 池 - ```bash - kubectl apply -f - <` (推荐) - - 在注解中加上 `eip.porter.kubesphere.io/v1alpha1: ` - -3. 在路由器上查看是否有对应的路由。如果有，那么连接这个路由器的任何主机应该都能通过EIP+ServicePort的方式访问了。 - - ``` - # show bgp all - - 10.11.11.11/32, ubest/mbest: 3/0 - *via 10.10.12.2, [20/0], 00:03:38, bgp-65001, external, tag 65000 - *via 10.10.12.3, [20/0], 00:03:38, bgp-65001, external, tag 65000 - *via 10.10.12.4, [20/0], 00:03:38, bgp-65001, external, tag 65000 - - ``` -4. 在集群上使用`kubectl get eip`观察当前集群 EIP 的使用情况 \ No newline at end of file diff --git a/doc/zh/eip_config.md b/doc/zh/eip_config.md new file mode 100644 index 000000000..bf95ffad5 --- /dev/null +++ b/doc/zh/eip_config.md @@ -0,0 +1,87 @@ +# Eip Configuration + +> [English](../eip_config.md) | 中文 + +Eip用于配置IP地址段，Porter会将其分配给LoadBalancer Service，后续通过`BGP/ARP/NDP`等协议发布Eip路由。Porter目前只支持IPv4地址, 对IPv6地支持也会马上完成。 + +下面Eip的例子展示了全部可用配置字段以及状态字段说明 + +```yaml +apiVersion: network.kubesphere.io/v1alpha2 +kind: Eip +metadata: + name: eip-sample-pool +spec: + address: 192.168.0.0/24 + protocol: layer2 + interface: eth0 + disable: false +status: + occupied: false + usage: 1 + poolSize: 256 + used: + "192.168.0.1": "default/test-svc" + firstIP: 192.168.0.0 + lastIP: 192.168.0.255 + ready: true + v4: true +``` + +## spec字段解释 + +* address + +`address`用于描述IP地址范围， 它可以有如下三种格式 + +```yaml +- ip e.g. 192.168.0.1 +- ip/net e.g. 192.168.0.0/24 +- ip1-ip2 e.g. 192.168.0.1-192.168.0.10 +``` + +**Note：IP地址段不可与其他已经创建的Eip重叠，否则创建资源会报错** + +* protocol + +`protocol`用于描述使用何种协议发布路由，合法的值有`layer2` 和 `bgp`. 当值为空时， 模式协议为`bgp`. + +* interface + +`interface`在`protocol`为`layer2`时才有意义， 它用于指示Porter在哪块网卡上监听ARP/NDP请求。 + +当Kubernetes集群中每个节点中网卡名字不同时， 你可以通过这种语法`interface: can_reach:192.168.1.1`指定网卡。 以上例子中， Porter通过查找到192.168.1.1的路由，获取路由中的第一块网卡。 + +* disable + +当值为`true`时， 新创建LoadBalancer Service时Porter将不会从这个Eip中分配地址， 但是不会影响已经创建的Service。 + +## status字段解释 + +* occupied + +此字段用于表示Eip中地址是否被分配使用完。 + +* usage 和 used + +`usage`用于表示Eip中已经分配了多少个地址； `used`用于表示哪个地址正在被哪个Service使用， key为IP地址，value为Service的`Namespace/Name`. + +* poolSize + +此字段用于表示Eip中总共有多少个地址 + +* firstIP + +此字段用于表示Eip中第一个IP地址 + +* lastIP + +此字段用于表示Eip中最后一个IP地址 + +* v4 + +此字段用于表示Eip的地址协议族 + +* ready + +此字段用于表示Eip关联的BGP/ARP/NDP相关程序是否初始化完毕 \ No newline at end of file diff --git a/doc/zh/index.md b/doc/zh/index.md new file mode 100644 index 000000000..04f04a0e0 --- /dev/null +++ b/doc/zh/index.md @@ -0,0 +1,18 @@ +## Install Porter + +* [deploy](./deploy.md) + +## Configure Porter + +* [bgp_config](./bgp_config.md) +* [eip_config](./eip_config.md) + +## Use Porter + +* [usage](./usage.md) +* [layer2](./layer2.md) +* [simulate_with_bird](./simulate_with_bird.md) + +## Porter VS. MetalLB + +* [compared_with_metallb](./compared_with_metallb.md) \ No newline at end of file diff --git a/doc/zh/layer2.md b/doc/zh/layer2.md index c90ecddc0..59d512f33 100644 --- a/doc/zh/layer2.md +++ b/doc/zh/layer2.md @@ -1,49 +1,116 @@ -# 与MetalLB Layer2的区别 -MetalLB目前Layer2的信息通告都是通过Daemonset中的speaker进行的，并且通过Memberlist维护所有speaker关系， -保证同一个Service EIP的ARP请求只会被一个speaker应答。这么做的原因有二：通过Daemonset保证speaker的高可用； -通过speaker分散应答ARP请求，减轻speaker压力。 +# Porter layer2模式 -考虑到Layer2场景下，其实ARP请求并不会是一个非常高频的请求，并且请求之后也会有cache，另外ARP请求本身就是一个广播 -请求，尽管MetalLB的部分speaker不应答请求，但还是会收到广播数据包，只是最后被speaker中Drop掉，相对于应答也并没有 -减轻什么压力。所以Porter最终实现Layer2将其ARP请求接受应答处理都放在Porter manager中， 通过manager的主备策略实现 -高可用，同时可以复用代码。 +现在大多数路由器都支持BGP，但是在实际应用过程中或多或少会由于某些原因而不便开放BGP功能， 例如安全合规， 或者用户使用的物理路由器实在太老不支持BGP， 那么这里可以通过配置Porter layer2模式用以达到类似的功能。 -在构造ARP应答包的时候，Porter会采用manager所在NODE的MAC应答， 这样可以避免一些云平台的ARP Spoof引起的丢包。 +## layer2 原理 -# Porter Layer2的使用 -为了支持Layer2，以及后续不同的LB策略，在EIP中引入字段lbTye，用以表示是使用bgp，layer2或者其他。 +当客户端通过IP访问服务端时，由于配置的Eip与Kubernetes集群处于同一二层网络， 路由器会通过发送ARP/NDP请求查找Eip对应的MAC地址，这个时候Porter会根据LoadBalancer Service的Endpoints应答Kubernetes Node对应MAC。ARP/NDP应答完成之后，后续客户端流量都会发往同一个Node。 -例如创建一个layer2的EIP +由于IP与MAC的一一对应关系， 在LoadBalancer Service的生命周期内只能应答同一个Node的MAC地址，除非Endpoints变化。 为了做到这一点， Porter采用Kubernetes自带的Leader Election功能， 通过它实现只会有一个副本应答ARP/NDP请求。 + +**限制：客户端通过Eip连接服务端时会存在单点故障， Eip的所有流量都会发往同一个Node** + + +## layer2的使用 + +在使用上， layer2模式较BGP模式简单很多，只需要配置layer2模式的Eip，并在创建工作负载时指定protocol为layer2 + +* 创建layer2 eip ```yaml kubectl apply -f - < Ubuntu 18.04.4 LTS 4.15.0-109-generic docker://19.3.6 +node3 Ready worker 18d v1.17.9 172.22.0.9 Ubuntu 18.04.4 LTS 4.15.0-108-generic docker://19.3.8 +node4 Ready worker 18d v1.17.9 172.22.0.10 Ubuntu 18.04.4 LTS 4.15.0-101-generic docker://19.3.8 +root@node1:~# +``` + +通过以上操作发现172.22.0.188下一跳即为172.22.0.3， 因为他们的MAC地址都相同，指向同一节点node1。 + + diff --git a/doc/zh/porter-chart.md b/doc/zh/porter-chart.md deleted file mode 100644 index 33fc34516..000000000 --- a/doc/zh/porter-chart.md +++ /dev/null @@ -1,298 +0,0 @@ -# 使用 Helm Chart 安装 Porter - -> [English](../porter-chart.md) | 中文 - -# 安装 Porter - -```bash -helm repo add test https://charts.kubesphere.io/test -helm repo update -helm install porter test/porter -``` - -# layer2模式 - -## 前提条件 - -- Kubernetes集群，版本1.17.3及以上 - -- 局域网内一台linux机器hostA，用于检测nginx的LoadBalancer - -## 配置layer2 - -```bash -$ cat << EOF > layer2.yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: Eip -metadata: - name: eip-sample-pool -spec: - # 修改ip地址段为实际环境的ip地址段。可以为单个地址或者是地址段 - address: 192.168.3.100 - protocol: layer2 - disable: false -EOF -$ kubectl apply -f layer2.yaml -eip.network.kubesphere.io/eip-sample-pool created -``` - -## 部署nginx - -在Kubernetes集群上: - -```bash -$ cat << EOF > nginx-layer2.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - lb.kubesphere.io/v1alpha1: porter - protocol.porter.kubesphere.io/v1alpha1: layer2 - name: nginx-service -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 8088 - targetPort: 80 -EOF -$ kubectl apply -f nginx-layer2.yaml -deployment.apps/nginx-deployment created -service/nginx-service created -$ kubectl get svc/nginx-service -default kubernetes ClusterIP 10.96.0.1 443/TCP 129m -default nginx-service LoadBalancer 10.100.5.90 192.168.3.100 8088:32063/TCP 50s -``` - -## 访问nginx服务 - -在hostA访问nginx - -```bash -$ curl 192.168.3.100:8088 -``` - -# BGP 模式 - -## 前提条件 - -- Kubernetes集群，版本1.17.3及以上。 - -- 开启BGP的路由器。在这里我们将在Centos7系统上安装bird，使用bird实现BGP路由功能。我们以Router称这台机器。 - -- 局域网内一台linux机器hostA，用于检测nginx的LoadBalancer - -## 网络图 - -```bash - ________________ ________________ ________________ -| | | | | | -| k8s cluster | <--------- | Router | <--------- | other host | -|_______________| |________________| |________________| -``` - -- Router在这里是一个路由器，实验中我们没有具有bgp功能的路由器，因此使用一台主机替代。 - -- 其他主机将包发送个Router，Router在将包发送给k8s cluster。 - -- k8s cluster需要使用BGP协议和Router建立连接，因此两者的as域必须不一样。 - -## Router 配置 - -在 Router 上安装 Bird - -```bash -$ yum install bird -$ systemctl enable bird -``` - -在 Router 上配置 BGP，如下 - -```bash -cat /etc/bird.conf -protocol kernel { - scan time 60; # Scan kernel routing table every 20 seconds - import none; # Default is import all - export all; # Default is export none - merge paths on; # Enable ECMP, this parameter requires at least bird 1.6 -} - -protocol device { - scan time 10; # Scan interfaces every 10 seconds -} - -protocol static { -} - -protocol bgp mymaster { - description "10.55.0.127"; # 本机ip地址 - local as 65001; # as域，必须和port-manager的as域不一样 - neighbor 10.55.0.124 port 17900 as 65000; # port-manager的as域不一样 - source address 10.55.0.127; # 本机ip地址 - import all; - export all; - enable route refresh off; - add paths on; -} -``` - -在Router上启动bird，并设置ipv4转发。 - -```bash -$ systemctl restart bird -$ sysctl -w net.ipv4.ip_forward=1 -``` - -在Router上查看配置是否生效,你会看到新添一条mymaster规则。 - -```bash -$ birdc show protocol -BIRD 1.6.8 ready. -name proto table state since info -kernel1 Kernel master up 18:01:55 -device1 Device master up 18:01:55 -static1 Static master up 18:01:55 -mymaster BGP master start 18:01:55 Active Socket: Connection refused -``` - -## 在 Porter 和 Router 上建立 BGP 连接 - -在Kubernetes上: - -```bash -$ cat << EOF > bgp.yaml -apiVersion: network.kubesphere.io/v1alpha1 -kind: Eip -metadata: - name: eip-sample-pool -spec: - # 修改ip地址段为实际环境的ip地址段。 - address: 10.55.0.100 - protocol: bgp - disable: false ---- -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpConf -metadata: - name: bgpconf-sample -spec: - # 设置porter的as域 - as : 65000 - routerID : 10.55.0.124 - port: 17900 ---- -apiVersion: network.kubesphere.io/v1alpha1 -kind: BgpPeer -metadata: - name: bgppeer-sample -spec: - # 设置需要建立连接的as域，这里使用route的as域 - config: - peerAs : 65001 - neighborAddress: 10.55.0.127 - addPaths: - sendMax: 10 -EOF -$ kubectl apply -f bgp.yaml -eip.network.kubesphere.io/eip-sample-pool created -bgpconf.network.kubesphere.io/bgpconf-sample created -bgppeer.network.kubesphere.io/bgppeer-sample created -``` - -在Router上查看是否建立连接,info信息显示Established表示建立连接。 - -```bash -$ birdc show protocol -BIRD 1.6.8 ready. -name proto table state since info -kernel1 Kernel master up 18:10:39 -device1 Device master up 18:10:39 -static1 Static master up 18:10:39 -mymaster BGP master up 18:15:45 Established -``` - -## 部署nginx - -```bash -$ cat << EOF > nginx-bgp.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment - labels: - app: nginx -spec: - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - lb.kubesphere.io/v1alpha1: porter - protocol.porter.kubesphere.io/v1alpha1: bgp - name: nginx-service -spec: - selector: - app: nginx - type: LoadBalancer - ports: - - name: http - port: 8088 - targetPort: 80 -EOF -$ kubectl apply -f nginx-bgp.yaml -deployment.apps/nginx-deployment created -service/nginx-service created -``` - -## 访问nginx服务 - -如果局域网内其他机器想要访问nginx，需要在设置路由。将包转发给Router。 - -```bash -$ #-host指单台机器，如果需要指定网段请使用-net -$ #192.168.3.100指应用服务的地址，这里使用nginx service地址 -$ #192.168.3.85指route地址。 -$ #eth0指网卡 -$ route add -host 192.168.3.100 gw 192.168.3.85 eth0 -``` - -```bash -$ curl 192.168.3.100:8088 -``` - diff --git a/doc/zh/router_config.md b/doc/zh/router_config.md deleted file mode 100644 index 311777274..000000000 --- a/doc/zh/router_config.md +++ /dev/null @@ -1,5 +0,0 @@ -# 路由器 BGP 配置参考 - -> [English](../router_config.md) | 中文 - - diff --git a/doc/zh/simulate_with_bird.md b/doc/zh/simulate_with_bird.md index 786e0e0b2..9699e78e1 100644 --- a/doc/zh/simulate_with_bird.md +++ b/doc/zh/simulate_with_bird.md @@ -2,174 +2,192 @@ > [English](../simulate_with_bird.md) | 中文 -> 本文是在[青云平台](https://www.qingcloud.com/)上做的实验，不同的平台有一些配置有区别。模拟的好处是能够不接触实际硬件的情况下体验Porter的功能，但和实际路由器还是有差别，模拟路由器有个默认的网卡（不用于路由），会导致报文回去的时候走这个默认路由。另外在配置模拟路由器时，有很多额外的细节参数，请按照本文所述的参数进行设置。 +作为普通开发者，日常工作中很难接触到硬件路由器，幸运地是很多软件可以提供物理路由器类似的功能，例如[bird](https://bird.network.cz/)、[gobgp](https://osrg.github.io/gobgp/)，它们同样能够协助我们开发。 ## 前提 -1. 已经拥有一个正常运行的k8s集群 -2. 确保用于模拟路由器的主机和k8s集群相互连通，包括集群中的bgp端口，和应用使用的端口。 -3. 有可供实验的公网ip - - -## 创建路由器 - -1. 在k8s所在的网络内创建一个主机，最小配置即可。进入主机安装bird，青云平台主机上默认只有bird 1.5，1.5不支持ECMP，要体验Porter的全部功能需要至少1.6，执行下面的脚本安装bird 1.6 - ``` - $sudo add-apt-repository ppa:cz.nic-labs/bird ##这里引入了Bird 1.6 - $sudo apt-get update - $sudo apt-get install bird - $sudo systemctl enable bird - ``` -2. 配置路由器的BGP服务。修改`/etc/bird/bird.conf`，添加如下参数： - ``` - protocol bgp mymaster { - description "192.168.1.4"; ##填本交换机ID，一般是主IP - local as 65001; ##填本地AS域，必须和k8s集群的AS不同 - neighbor 192.168.1.5 port 17900 as 65000; ##填master节点IP和 AS域 - source address 192.168.1.4; ##填本交换机IP - import all; - export all; - enable route refresh off; #由于bird1.6的BGP协议较低，和Porter的bgp连接会将多路由变成单个路由，这个参数能够作为一个workaround修正这个问题。 - add paths on; #这个参数开启之后，就可以收到porter发来的多个路由并同时存在而不会覆盖。 - } - ``` - 上述参数给模拟路由器配置了一个邻居，邻居即是集群主节点，**这里假设了porter的controller部署在了主节点，如果不想限制porter部署在主节点，或者master不能部署pod，那么在这里需要按照上述配置，将所有可能的邻居节点按照上述规则添加到这个配置文件中**。修改该文件中的kernel部分，将其中的export all的注释取消，并开启ECMP功能。修改为： - ``` - protocol kernel { - scan time 60; - import none; - export all; # Actually insert routes into the kernel routing table - merge paths on; #开启ECMP功能，这个参数至少需要 bird 1.6 - } - - ``` -3. 重启bird - ```bash - $sudo systemctl restart bird - ``` - -4. 配置公网ip。在青云控制台上申请一个**内部绑定**的公网IP，注意必须是一个内部绑定的IP，如果是外部绑定的话是无法感知这个公网ip的。将这个ip绑定到模拟路由器所在主机上。绑定完成即可，不需要按照青云文档进行后续的操作。 - -5. 观察网卡。青云平台内部绑定的IP绑定到主机上时，会在主机上创建一个新的网卡（一般是eth1）,登录主机执行`ip a`查看这个网卡是否启动（状态是否为`UP`），如果没有启动，执行`ip link set up eth1`。eth1是这个ip的入口网卡。 - -6. 打开模拟路由器的端口转发并关闭包过滤 - ``` - sysctl -w net.ipv4.ip_forward=1 - sysctl -w net.ipv4.conf.all.rp_filter=0 - sysctl -w net.ipv4.conf.eth1.rp_filter=0 - sysctl -w net.ipv4.conf.eth0.rp_filter=0 - ``` -7. 配置防火墙。在青云控制台上打开一些测试端口，如8000-30000等。 - -8. 配置路由回路。由于模拟路由器的默认网卡是eth0，在集群返回ip包之后，默认会从eth0发出，而用户访问这个公网ip是从eth1进来的，这样就会导致信息发送失败，所以需要将从绑定的IP发来的包导流到eth1。在浏览器上访问这个地址，同时在模拟路由器上使用`tcpdump -i eth1`抓包，观察上层路由地址，如： - - ```bash - root@i-7bwamgny:~# tcpdump -i eth1 - tcpdump: verbose output suppressed, use -v or -vv for full protocol decode - listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes - 14:24:07.401555 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475097 ecr 0], length 0 - 14:24:07.403573 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475100 ecr 0], length 0 - 14:24:07.654341 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532475350 ecr 0], length 0 - 14:24:10.400770 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478097 ecr 0], length 0 - 14:24:10.404100 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478100 ecr 0], length 0 - 14:24:10.658557 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532478351 ecr 0], length 0 - 14:24:16.401591 IP 139.198.254.4.1395 > 139.198.121.228.omniorb: Flags [S], seq 3677905607, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484098 ecr 0], length 0 - 14:24:16.404605 IP 139.198.254.4.1396 > 139.198.121.228.omniorb: Flags [S], seq 2462558694, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484101 ecr 0], length 0 - 14:24:16.656750 IP 139.198.254.4.1397 > 139.198.121.228.omniorb: Flags [S], seq 1471601642, win 64240, options [mss 1394,nop,wscale 8,sackOK,TS val 532484351 ecr 0], length 0 - - ``` - 上述打印输出中，`139.198.121.228`是绑定的ip，左边即上层路由器的地址。获取到这个地址之后，通过路由策略配置回去的规则： - ```bash - sudo ip rule add from 139.198.254.4/32 lookup 101 #返回这个ip的包走路由表101 - sudo ip route replace default dev eth1 table 101 #路由表101的默认网卡是eth1 - ``` - 实际物理路由器不需要配置上述规则，因为路由器知道如何正确配置这个ip。**如果需要从多个ip地址访问测试ECMP，那么这些IP也需要相同的步骤** +1. 拥有一个正常运行的Kubernetes集群 +2. Porter已正确安装 +3. 用于模拟路由器的主机和Kubernetes集群网络互通 + +## 创建并配置路由器 + +1. 安装bird。 -9. 这样模拟路由器就配置完成了，可以执行`birdc show protocol`查看连接信息。 - -> 注：如果连接主机的方式是通过公网IP的方式，那么执行上述操作之后**有可能**会导致SSH连接断掉（当且仅当SSH的公网IP和你测试用的公网IP在青云网络中都会NAT成上述的139.198.254.4/32）。断掉之后可以使用青云网页上的VNC的方式。建议使用VPN的方式连接。下面在k8s集群中的操作会同样的影响。 - -## 配置插件 -> 所有的操作都在k8s集群的主节点中 - -1. 获取yaml文件 - ``` - wget https://github.com/kubesphere/porter/releases/download/v0.1.1/porter.yaml - ``` -2. 修改yaml文件中的configmap `bgp-cfg`，请按照配置这个文件，并且需要和刚才模拟器配置相对应。 -3. 配置公网ip回路规则。和模拟路由器的问题一致，公网ip导流至集群中之后，ip包发出默认都是eth0，eth0会将此包丢弃，需要将此ip包导向模拟路由器。**这一步需要在k8s所有节点上配置，因为实际的服务可能部署在任何节点。** - ```bash - sudo ip rule add to 139.198.254.0/24 lookup 101 #返回这个ip的包走路由表101 - sudo ip route replace default via 192.168.98.5 dev eth0 table 101 #路由表101的默认网关是192.168.98.5这个模拟路由器 - ``` - 上面的`192.168.98.5`即模拟路由器的地址，模拟路由器上已经配置了一条回路规则，所以此包就不会被丢弃了。实际k8s集群不需要配置，因为k8s集群的默认网关就是这个路由器。 - -4. 安装porter到集群中，`kubectl apply -f porter.yaml` -5. 添加一个EIP到集群中。 - ```bash - kubectl apply -f - < 使用这个样例之前需先替换里面的EIP - ``` - kubectl apply -f service.yaml - ``` -7. 检查一下Porter日志和EIP events，如果没问题，就可以按照Service中的EIP和其端口访问服务了。 - ```bash - kubectl logs -n porter-system controller-manager-0 -c manager - kubectl describe eip eip-sample ##观察是否有对应的event - ``` -8. 检查模拟路由器上，是否有两个等价路由： - ```bash - root@i-7bwamgny:~# ip route - default via 192.168.98.1 dev eth0 - 139.198.121.228 proto bird - nexthop via 192.168.98.2 dev eth0 weight 1 - nexthop via 192.168.98.4 dev eth0 weight 1 - ``` - -## 测试ECMP带来的Load Balancing 功能 -> 注，模拟路由器所在的主机内核版本要高于3.6，青云平台默认内核为4.4，使用的ECMP Hash算法是`L3`的，ECMP只会根据源IP调整访问的路由。4.12以上的内核支持设置`L4`的。，可以设置`sysctl net.ipv4.fib_multipath_hash_policy 1`改变负载均衡hash算法，那么利用curl访问这个eip也可以实现测试负载均衡的效果。 - -实际路由器只需要开启ECMP功能就可以实现负载均衡，为了测试负载均衡有效，需要从不同的源IP访问这个EIP，同时在Pod分布的节点上用tcpdump观察是否有流量。 -1. 首先观察Pod所在节点 +`1.5不支持ECMP，要体验Porter的全部功能需要至少1.6`, 在ubuntu中可以执行下面的脚本安装bird 1.6 + +``` +$sudo add-apt-repository ppa:cz.nic-labs/bird ##这里引入了bird 1.6 +$sudo apt-get update +$sudo apt-get install bird +$sudo systemctl enable bird +``` + +2. 配置bird + +bird的配置文件为`/etc/bird/bird.conf`, 具体配置可以参考[bird官方文档](https://bird.network.cz/?get_doc&f=bird.html&v=16) + +* 配置router id + +`router id`格式为合法的ip地址， 请根据实际环境修改 + +``` +router id 172.22.0.2; +``` + +* 配置bgp neighbor + +根据`Porter Manager`实际部署节点配置bgp neighbor， **如有多个请对应添加多个neighbor**。 + +``` +protocol bgp neighbor1 { + local as 65001; #填本地AS域，必须和Kubernetes集群的AS不同 + neighbor 172.22.03 port 17900 as 65000; ##填master节点IP和 AS域 + source address 172.22.0.2; #填本交换机IP + import all; + export all; + enable route refresh off; #由于bird1.6的bgp较低，和Porter的bgp连接会将多路由变成单个路由，这个参数能够作为一个workaround修正这个问题。 + add paths on; #这个参数开启之后，就可以收到porter发来的多个路由并同时存在而不会覆盖。 +} +``` + +* 配置bird kernel + +添加`export all;` 和 `merge paths on;` 用于向linux kernel添加路由。 + +``` +protocol kernel { + scan time 60; + import none; + export all; + merge paths on; #开启ECMP功能，这个参数至少需要 bird 1.6 +} +``` + +3. 重启bird + ```bash -kubectl get pod -o wide +$sudo systemctl restart bird ``` -2. 观察各个节点是否设置了引流规则 + +4. 确认bird + +通过以下命令查看bird配置是否正常启动。如果状态为非`active`, 可以执行`journalctl -f -u bird`查看错误。 + ```bash -root@master-k8s:~# ip rule -0: from all lookup local -32763: from all to 139.198.121.228 lookup 101 +$sudo systemctl status bird ``` -3. 然后在这些节点上运行tcpdump -i eth0 port $port, $port就是服务对外暴露的端口，上面的例子中就是8088。 -4. 从不同的IP访问这个EIP，观察在这些节点上是否有对应的流量，如果都有，那么负载均衡就没有问题。 +## 配置Porter + +* 配置BgpConf + +请参考[bgp_config](./bgp_config.md)， 按照自己需求修改下面配置 + +```yaml +kubectl apply -f - < 0 { + return fmt.Errorf("eip is inusing") + } + + if e.Spec.Protocol == constant.PorterProtocolLayer2 { + speaker.UnRegisteSpeaker(e.Spec.Interface) + } + + return nil +} + +func (i *IPAM) AssignIP(args IPAMArgs) (IPAMResult, error) { + eips := &networkv1alpha2.EipList{} + err := i.List(context.Background(), eips) + if err != nil { + return IPAMResult{}, err + } + + err = fmt.Errorf("no avliable eip") + var result IPAMResult + + for _, eip := range eips.Items { + clone := eip.DeepCopy() + addr := args.assignIPFromEip(clone) + if addr != "" { + if !reflect.DeepEqual(clone, eip) { + err = i.Client.Status().Update(context.Background(), clone) + ctrl.Log.Info("assignIP update eip", "eip", clone.Status) + } + + result.Addr = addr + result.Eip = eip.Name + result.Protocol = eip.GetProtocol() + result.Sp = speaker.GetSpeaker(eip.GetSpeakerName()) + + if result.Sp == nil { + err = fmt.Errorf("layer2 eip speaker not ready") + } + + break + } + } + + i.log.Info("assignIP", + "args", args, + "result", result, + "err", err) + + return result, err +} + +func (a IPAMArgs) assignIPFromEip(eip *networkv1alpha2.Eip) string { + for addr, svcs := range eip.Status.Used { + tmp := strings.Split(svcs, ";") + for _, svc := range tmp { + if svc == a.Key { + return addr + } + } + } + + if a.Protocol != eip.GetProtocol() { + return "" + } + + if eip.Name != a.Eip && a.Eip != "" { + return "" + } + + if eip.Spec.Disable || !eip.Status.Ready { + return "" + } + + ip := net.ParseIP(a.Addr) + offset := 0 + if ip != nil { + offset = eip.IPToOrdinal(ip) + if offset < 0 { + return "" + } + } + + for ; offset < eip.Status.PoolSize; offset++ { + addr := cnet.IncrementIP(*cnet.ParseIP(eip.Status.FirstIP), big.NewInt(int64(offset))).String() + tmp, ok := eip.Status.Used[addr] + if !ok { + if eip.Status.Used == nil { + eip.Status.Used = make(map[string]string) + } + eip.Status.Used[addr] = a.Key + eip.Status.Usage++ + if eip.Status.Usage == eip.Status.PoolSize { + eip.Status.Occupied = true + } + return addr + } else { + if ip != nil { + eip.Status.Used[addr] = fmt.Sprintf("%s;%s", tmp, a.Key) + return addr + } + } + } + + return "" +} + +// look up by key in IPAMArgs +func (a IPAMArgs) unAssignIPFromEip(eip *networkv1alpha2.Eip, peek bool) string { + for addr, svcs := range eip.Status.Used { + tmp := strings.Split(svcs, ";") + for _, svc := range tmp { + if svc == a.Key { + if !peek { + if len(tmp) == 1 { + delete(eip.Status.Used, addr) + eip.Status.Usage-- + if eip.Status.Usage != eip.Status.PoolSize { + eip.Status.Occupied = false + } + } else { + eip.Status.Used[addr] = strings.Join(util.RemoveString(tmp, a.Key), ";") + } + } + + return addr + } + } + } + + return "" +} + +func (i *IPAM) UnAssignIP(args IPAMArgs, peek bool) (IPAMResult, error) { + var result IPAMResult + + eips := &networkv1alpha2.EipList{} + err := i.List(context.Background(), eips) + if err != nil { + return result, err + } + + for _, eip := range eips.Items { + clone := eip.DeepCopy() + addr := args.unAssignIPFromEip(clone, peek) + if addr != "" { + if !reflect.DeepEqual(clone, eip) && !peek { + err = i.Client.Status().Update(context.Background(), clone) + ctrl.Log.Info("unAssignIP update eip", "eip", clone.Status) + } + + result.Addr = addr + result.Eip = eip.Name + result.Protocol = eip.GetProtocol() + result.Sp = speaker.GetSpeaker(eip.GetSpeakerName()) + + if result.Sp == nil { + err = fmt.Errorf("layer2 eip speaker not ready") + } + break + } + } + + i.log.Info("unAssignIP", + "args", args, + "peek", peek, + "result", result, + "err", err) + + return result, err +} diff --git a/pkg/controllers/ipam/ipam_test.go b/pkg/controllers/ipam/ipam_test.go new file mode 100644 index 000000000..a0be92597 --- /dev/null +++ b/pkg/controllers/ipam/ipam_test.go @@ -0,0 +1,291 @@ +package ipam + +import ( + "fmt" + "testing" + + "github.com/kubesphere/porter/api/v1alpha2" + "github.com/kubesphere/porter/pkg/constant" + "github.com/kubesphere/porter/pkg/speaker" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestIpam(t *testing.T) { + RegisterFailHandler(Fail) + log := zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) + RunSpecs(t, "Ipam Suite") +} + +var _ = BeforeSuite(func() { + speaker.RegisteSpeaker(e.GetSpeakerName(), speaker.NewFake()) + speaker.RegisteSpeaker(e2.GetSpeakerName(), speaker.NewFake()) + IPAMAllocator = &IPAM{ + Client: nil, + log: nil, + EventRecorder: nil, + } +}) + +var ( + e = v1alpha2.Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "testeip1", + }, + Spec: v1alpha2.EipSpec{ + Address: "192.168.1.0/24", + }, + Status: v1alpha2.EipStatus{}, + } + + e2 = v1alpha2.Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "testeip2", + }, + Spec: v1alpha2.EipSpec{ + Address: "192.168.2.0/24", + Protocol: constant.PorterProtocolLayer2, + Interface: "eth0", + }, + Status: v1alpha2.EipStatus{}, + } +) + +var _ = Describe("IPAMArgs", func() { + It("should be false, when IPAMResult.Addr is empty", func() { + a := IPAMArgs{ + Key: "testsvc", + Addr: "", + Eip: "", + Protocol: "", + } + r := IPAMResult{ + Addr: "", + Eip: "", + Protocol: "", + Sp: nil, + } + Expect(a.ShouldUnAssignIP(r)).ShouldNot(BeTrue()) + }) + + It("should be true, when addr not equal", func() { + a := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.0.1", + Eip: "", + Protocol: "", + } + r := IPAMResult{ + Addr: "192.168.0.2", + Eip: "", + Protocol: "", + Sp: nil, + } + Expect(a.ShouldUnAssignIP(r)).Should(BeTrue()) + }) + + It("should be true, when eip not equal", func() { + a := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.0.1", + Eip: "testeip", + Protocol: "", + } + r := IPAMResult{ + Addr: "192.168.0.1", + Eip: "testeip2", + Protocol: "", + Sp: nil, + } + Expect(a.ShouldUnAssignIP(r)).Should(BeTrue()) + }) + + It("should be true, when protocol not equal", func() { + a := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.0.1", + Eip: "testeip", + Protocol: constant.PorterProtocolLayer2, + } + r := IPAMResult{ + Addr: "192.168.0.1", + Eip: "testeip", + Protocol: constant.PorterProtocolBGP, + Sp: nil, + } + Expect(a.ShouldUnAssignIP(r)).Should(BeTrue()) + }) + + It("should be false, when all fields equal", func() { + a := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.0.1", + Eip: "testeip", + Protocol: constant.PorterProtocolBGP, + } + r := IPAMResult{ + Addr: "192.168.0.1", + Eip: "testeip", + Protocol: constant.PorterProtocolBGP, + Sp: nil, + } + Expect(a.ShouldUnAssignIP(r)).ShouldNot(BeTrue()) + }) +}) + +var _ = Describe("IPAM", func() { + It("Add Eip", func() { + IPAMAllocator.updateEip(&e) + Expect(e.Status.PoolSize).Should(Equal(256)) + Expect(e.Status.Usage).Should(Equal(0)) + Expect(e.Status.Occupied).Should(Equal(false)) + Expect(e.Status.Ready).Should(Equal(true)) + Expect(e.Status.V4).Should(Equal(true)) + Expect(e.Status.FirstIP).Should(Equal("192.168.1.0")) + Expect(e.Status.LastIP).Should(Equal("192.168.1.255")) + + IPAMAllocator.updateEip(&e2) + Expect(e2.Status.PoolSize).Should(Equal(256)) + Expect(e2.Status.Usage).Should(Equal(0)) + Expect(e2.Status.Occupied).Should(Equal(false)) + Expect(e2.Status.Ready).Should(Equal(true)) + Expect(e2.Status.V4).Should(Equal(true)) + Expect(e2.Status.FirstIP).Should(Equal("192.168.2.0")) + Expect(e2.Status.LastIP).Should(Equal("192.168.2.255")) + }) + + Context("Assign ip", func() { + Context("eip unusable", func() { + It("eip not ready", func() { + e.Status.Ready = false + + addr := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.1.1", + Eip: "", + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("")) + + e.Status.Ready = true + }) + + It("eip not enabled", func() { + e.Spec.Disable = true + + addr := IPAMArgs{ + Key: "testsvc", + Addr: "192.168.1.1", + Eip: "", + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("")) + + e.Spec.Disable = false + }) + }) + + When("IPAMArgs incorrect", func() { + It("has no protocol", func() { + addr := IPAMArgs{ + Key: "testsvc", + Addr: "", + Eip: "", + Protocol: "", + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("")) + }) + + It("eip and protocol not match", func() { + addr := IPAMArgs{ + Key: "testsvc", + Addr: "", + Eip: e2.Name, + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e2) + Expect(addr).Should(Equal("")) + }) + }) + + When("IPAMArgs correct", func() { + It("Specify IP address", func() { + addr := IPAMArgs{ + Key: "testsvc255", + Addr: "192.168.1.255", + Eip: "", + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("192.168.1.255")) + Expect(len(e.Status.Used)).Should(Equal(1)) + Expect(e.Status.Occupied).Should(Equal(false)) + Expect(e.Status.Usage).Should(Equal(1)) + }) + + It("The address should be the same if it is reassigned using the same key.", func() { + addr := IPAMArgs{ + Key: "testsvc255", + Addr: "", + Eip: "", + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("192.168.1.255")) + Expect(len(e.Status.Used)).Should(Equal(1)) + Expect(e.Status.Occupied).Should(Equal(false)) + Expect(e.Status.Usage).Should(Equal(1)) + }) + + It("Assigning ip addresses cyclically, knowing that eip is running out.", func() { + addr := "" + for i := 0; i < 255; i++ { + addr = IPAMArgs{ + Key: fmt.Sprintf("testsvc%d", i), + Addr: "", + Eip: e.Name, + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal(fmt.Sprintf("192.168.1.%d", i))) + Expect(len(e.Status.Used)).Should(Equal(i + 2)) + Expect(e.Status.Usage).Should(Equal(i + 2)) + } + Expect(e.Status.Occupied).Should(Equal(true)) + + By("eip is full") + addr = IPAMArgs{ + Key: "testsvc256", + Addr: "", + Eip: "", + Protocol: constant.PorterProtocolBGP, + }.assignIPFromEip(&e) + Expect(addr).Should(Equal("")) + }) + }) + }) + + It("unAssign ip", func() { + addr := "" + for i := 0; i < 255; i++ { + addr = IPAMArgs{ + Key: fmt.Sprintf("testsvc%d", i), + }.unAssignIPFromEip(&e, false) + Expect(addr).Should(Equal(fmt.Sprintf("192.168.1.%d", i))) + Expect(len(e.Status.Used)).Should(Equal(256 - i - 1)) + Expect(e.Status.Usage).Should(Equal(256 - i - 1)) + Expect(e.Status.Occupied).Should(Equal(false)) + } + }) + + It("delete eip", func() { + Expect(IPAMAllocator.removeEip(&e)).Should(HaveOccurred()) + IPAMArgs{ + Key: "testsvc255", + }.unAssignIPFromEip(&e, false) + Expect(IPAMAllocator.removeEip(&e)).ShouldNot(HaveOccurred()) + }) +}) diff --git a/pkg/controllers/lb/controller.go b/pkg/controllers/lb/controller.go new file mode 100644 index 000000000..854700637 --- /dev/null +++ b/pkg/controllers/lb/controller.go @@ -0,0 +1,389 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lb + +import ( + "context" + "fmt" + "github.com/projectcalico/libcalico-go/lib/set" + "math/rand" + "reflect" + + "github.com/kubesphere/porter/pkg/constant" + "github.com/kubesphere/porter/pkg/controllers/ipam" + "github.com/kubesphere/porter/pkg/util" + "github.com/kubesphere/porter/pkg/validate" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch + +// ServiceReconciler reconciles a Service object +type ServiceReconciler struct { + client.Client + record.EventRecorder +} + +func getActiveEndpointNode(ep *corev1.Endpoints) set.Set { + active := make([]string, 0) + for _, subnet := range ep.Subsets { + for _, addr := range subnet.Addresses { + active = append(active, *addr.NodeName) + } + } + if len(active) <= 0 { + return nil + } + return set.FromArray(active) +} + +func (r *ServiceReconciler) SetupWithManager(mgr ctrl.Manager) error { + p := predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + return validate.IsPorterService(e.ObjectNew) + }, + CreateFunc: func(e event.CreateEvent) bool { + return validate.IsPorterService(e.Object) + }, + } + + // Watch for changes to Service + //return ctl.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForObject{}, p) + ctl, err := ctrl.NewControllerManagedBy(mgr). + For(&corev1.Service{}). + WithEventFilter(p). + Named("LBController"). + Build(r) + if err != nil { + return err + } + + //endpoints + p = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + svc := &corev1.Service{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: e.MetaOld.GetNamespace(), Name: e.MetaOld.GetName()}, svc) + if err != nil { + return true + } + + if !validate.IsPorterService(svc) { + return false + } + + oldSet := getActiveEndpointNode(e.ObjectOld.(*corev1.Endpoints)) + newSet := getActiveEndpointNode(e.ObjectNew.(*corev1.Endpoints)) + if newSet == nil && oldSet == nil { + return false + } else if newSet == nil || oldSet == nil { + return true + } + return !oldSet.Equals(newSet) + }, + CreateFunc: func(e event.CreateEvent) bool { + svc := &corev1.Service{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: e.Meta.GetNamespace(), Name: e.Meta.GetName()}, svc) + if err != nil { + return true + } + return validate.IsPorterService(svc) + }, + } + return ctl.Watch(&source.Kind{Type: &corev1.Endpoints{}}, &handler.EnqueueRequestForObject{}, p) +} + +func (r *ServiceReconciler) callSetLoadBalancer(result ipam.IPAMResult, svc *corev1.Service) ([]string, error) { + nodesIP, err := r.getServiceNodes(svc) + if err != nil { + return nil, err + } + + svcIP := result.Addr + + var announceNodes []string + if result.Protocol == constant.PorterProtocolLayer2 { + if len(nodesIP) == 0 { + return nil, result.Sp.DelBalancer(svcIP) + } + + index := rand.Int() % len(nodesIP) + found := false + preNodeIP, ok := svc.Annotations[constant.PorterLayer2Annotation] + if ok { + for i, nodeIP := range nodesIP { + if nodeIP == preNodeIP { + index = i + found = true + break + } + } + } + + if !found { + if svc.Annotations == nil { + svc.Annotations = make(map[string]string) + } + svc.Annotations[constant.PorterLayer2Annotation] = nodesIP[index] + + err = r.Update(context.Background(), svc) + if err != nil { + return nil, err + } + } + + announceNodes = append(announceNodes, nodesIP[index]) + } else { + announceNodes = append(announceNodes, nodesIP...) + } + + err = result.Sp.SetBalancer(svcIP, announceNodes) + ctrl.Log.Info("callSetLoadBalancer", "result", result, + "announceNodes", announceNodes, "avaliableNodes", nodesIP, "err", err) + return announceNodes, err + +} + +func (r *ServiceReconciler) callDelLoadBalancer(result ipam.IPAMResult, svc corev1.Service) error { + if len(svc.Status.LoadBalancer.Ingress) <= 0 { + return nil + } + + ip := svc.Status.LoadBalancer.Ingress[0].IP + + err := result.Sp.DelBalancer(ip) + ctrl.Log.Info("callDelLoadBalancer", "result", result, + "IngressIP", ip, "err", err) + return err +} + +const ( + ReasonDeleteLoadBalancer = "deleteLoadBalancer" + ReasonAddLoadBalancer = "addLoadBalancer" + AddLoadBalancerMsg = "success to add nexthops %v" + AddLoadBalancerFailedMsg = "failed to add nexthops %v, err=%v" + DelLoadBalancerMsg = "loadbalancer ip changed from %s to %s" + DelLoadBalancerFailedMsg = "speaker del loadbalancer failed, err=%v" +) + +func (r *ServiceReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + var ( + result ipam.IPAMResult + ) + + log := ctrl.Log.WithValues("service", req.NamespacedName) + + svc := &corev1.Service{} + err := r.Get(context.TODO(), req.NamespacedName, svc) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + args := r.constructIPAMArgs(svc) + result, err = ipam.IPAMAllocator.UnAssignIP(args, true) + if err != nil { + return ctrl.Result{}, err + } + + if util.IsDeletionCandidate(svc, constant.FinalizerName) { + if result.ShouldUnAssignIP() { + err = r.callDelLoadBalancer(result, *svc) + if err != nil { + r.Event(svc, corev1.EventTypeWarning, ReasonDeleteLoadBalancer, fmt.Sprintf(DelLoadBalancerFailedMsg, err)) + return ctrl.Result{}, err + } + } + + if result.ShouldUnAssignIP() { + _, err = ipam.IPAMAllocator.UnAssignIP(args, false) + if err != nil { + return ctrl.Result{}, err + } + } + + controllerutil.RemoveFinalizer(svc, constant.FinalizerName) + err = r.Update(context.Background(), svc) + log.Info("RemoveFinalizer", "finalizer", svc.Finalizers, "err", err) + return ctrl.Result{}, err + } + + if util.NeedToAddFinalizer(svc, constant.FinalizerName) { + controllerutil.AddFinalizer(svc, constant.FinalizerName) + err := r.Update(context.Background(), svc) + log.Info("AddFinalizer", "finalizer", svc.Finalizers, "err", err) + if err != nil { + return ctrl.Result{}, err + } + } + + clone := svc.DeepCopy() + clone.Status.LoadBalancer.Ingress = nil + + // Check if the IP address specified by the service should be changed. + if args.ShouldUnAssignIP(result) { + err = r.callDelLoadBalancer(result, *svc) + if err != nil { + r.Event(svc, corev1.EventTypeWarning, ReasonDeleteLoadBalancer, fmt.Sprintf(DelLoadBalancerFailedMsg, err)) + return ctrl.Result{}, err + } + + r.Event(svc, corev1.EventTypeNormal, ReasonDeleteLoadBalancer, fmt.Sprintf(DelLoadBalancerMsg, args.Addr, result.Addr)) + _, err = ipam.IPAMAllocator.UnAssignIP(args, false) + if err != nil { + return ctrl.Result{}, err + } + + result.Clean() + } + + if result.ShouldAssignIP() { + result, err = ipam.IPAMAllocator.AssignIP(args) + if err != nil { + return ctrl.Result{}, err + } + } + + clone.Status.LoadBalancer.Ingress = append(clone.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{ + IP: result.Addr, + }) + + nodes, err := r.callSetLoadBalancer(result, clone) + if err != nil { + r.Event(svc, corev1.EventTypeWarning, ReasonAddLoadBalancer, fmt.Sprintf(AddLoadBalancerFailedMsg, nodes, err)) + return ctrl.Result{}, err + } + + r.Event(svc, corev1.EventTypeNormal, ReasonAddLoadBalancer, fmt.Sprintf(AddLoadBalancerMsg, nodes)) + if reflect.DeepEqual(svc, clone) { + return ctrl.Result{}, nil + } + + err = r.Status().Update(context.Background(), clone) + log.Info("UpdateIngress", "Ingress", clone.Status.LoadBalancer.Ingress, "err", err) + return ctrl.Result{}, err +} + +func (r *ServiceReconciler) constructIPAMArgs(svc *corev1.Service) ipam.IPAMArgs { + args := ipam.IPAMArgs{} + + args.Key = types.NamespacedName{ + Name: svc.Name, + Namespace: svc.Namespace, + }.String() + + if svc.Annotations != nil { + if ip, ok := svc.Annotations[constant.PorterEIPAnnotationKey]; ok { + args.Addr = ip + } + + if eip, ok := svc.Annotations[constant.PorterEIPAnnotationKeyV1Alpha2]; ok { + args.Eip = eip + } + + if protocol, ok := svc.Annotations[constant.PorterProtocolAnnotationKey]; ok { + args.Protocol = protocol + } else { + args.Protocol = constant.PorterProtocolBGP + } + } + + if svc.Spec.LoadBalancerIP != "" { + args.Addr = svc.Spec.LoadBalancerIP + } + + return args +} + +// The caller should check if the slice is empty. +func (r *ServiceReconciler) getServiceNodes(svc *corev1.Service) ([]string, error) { + endpoints := &corev1.Endpoints{} + + err := r.Get(context.TODO(), types.NamespacedName{Namespace: svc.GetNamespace(), Name: svc.GetName()}, endpoints) + if err != nil { + return nil, err + } + + active := make([]string, 0) + for _, subnet := range endpoints.Subsets { + for _, addr := range subnet.Addresses { + active = append(active, *addr.NodeName) + } + } + if len(active) <= 0 { + return nil, nil + } + + nodeIPs, err := r.getNodeIPs() + if err != nil { + return nil, err + } + + result := make([]string, 0) + if svc.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyTypeLocal { + set.FromArray(active).Iter(func(item interface{}) error { + result = append(result, nodeIPs[item.(string)]) + return nil + }) + } else { + for _, nodeIP := range nodeIPs { + result = append(result, nodeIP) + } + } + + return result, nil +} + +func (r *ServiceReconciler) getNodeIPs() (map[string]string, error) { + nodeList := &corev1.NodeList{} + + err := r.List(context.TODO(), nodeList) + if err != nil { + return nil, err + } + + result := make(map[string]string) + for _, node := range nodeList.Items { + ip := util.GetNodeIP(node) + if ip != nil { + result[node.Name] = ip.String() + } + } + return result, nil +} + +func SetupServiceReconciler(mgr ctrl.Manager) error { + lb := &ServiceReconciler{ + Client: mgr.GetClient(), + EventRecorder: mgr.GetEventRecorderFor("PorterLB Manager"), + } + err := lb.SetupWithManager(mgr) + return err +} diff --git a/pkg/controllers/lb/suite_test.go b/pkg/controllers/lb/suite_test.go new file mode 100644 index 000000000..4be65c290 --- /dev/null +++ b/pkg/controllers/lb/suite_test.go @@ -0,0 +1,533 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lb + +import ( + "context" + "path/filepath" + "testing" + "time" + + networkv1alpha2 "github.com/kubesphere/porter/api/v1alpha2" + "github.com/kubesphere/porter/pkg/constant" + "github.com/kubesphere/porter/pkg/controllers/ipam" + "github.com/kubesphere/porter/pkg/manager" + "github.com/kubesphere/porter/pkg/manager/client" + "github.com/kubesphere/porter/pkg/speaker" + "github.com/kubesphere/porter/pkg/util" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var testEnv *envtest.Environment +var stopCh chan struct{} + +var ( + node1 = &corev1.Node{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Spec: corev1.NodeSpec{}, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: "192.168.0.1", + }, + }, + }, + } + + node2 = &corev1.Node{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + }, + Spec: corev1.NodeSpec{}, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: "192.168.0.2", + }, + }, + }, + } + + eip = &networkv1alpha2.Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "testeip", + }, + Spec: networkv1alpha2.EipSpec{ + Address: "10.0.0.1/24", + }, + Status: networkv1alpha2.EipStatus{}, + } + + eipLayer2 = &networkv1alpha2.Eip{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "testeip2", + }, + Spec: networkv1alpha2.EipSpec{ + Address: "10.0.2.1/24", + Protocol: constant.PorterProtocolLayer2, + Interface: "eth0", + }, + Status: networkv1alpha2.EipStatus{}, + } + + svc = &corev1.Service{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "testsvc", + Namespace: "default", + Annotations: map[string]string{ + constant.PorterAnnotationKey: constant.PorterAnnotationValue, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Ports: []corev1.ServicePort{ + { + Port: 80, + TargetPort: intstr.FromInt(80), + }, + }, + }, + Status: corev1.ServiceStatus{}, + } + + endpoints = &corev1.Endpoints{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: svc.Name, + Namespace: svc.Namespace, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "192.168.0.1", + NodeName: &node1.Name, + }, + }, + NotReadyAddresses: nil, + Ports: []corev1.EndpointPort{ + { + Port: 80, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + } + + bgpFakeSpeak = speaker.NewFake() + layer2FakeSpeak = speaker.NewFake() +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + stopCh = make(chan struct{}) + log := zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) + + RunSpecsWithDefaultAndCustomReporters(t, + "LB Controller Suite", + []Reporter{printer.NewlineReporter{}}) + +} + +var _ = BeforeSuite(func(done Done) { + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(cfg).ToNot(BeNil()) + + // +kubebuilder:scaffold:scheme + + mgr, err := manager.NewManager(cfg, manager.NewGenericOptions()) + Expect(err).ToNot(HaveOccurred()) + Expect(mgr).ToNot(BeNil()) + + // Setup all Controllers + err = ipam.SetupIPAM(mgr) + Expect(err).ToNot(HaveOccurred()) + + err = SetupServiceReconciler(mgr) + Expect(err).ToNot(HaveOccurred()) + + go func() { + err := mgr.Start(stopCh) + if err != nil { + ctrl.Log.Error(err, "failed to start manager") + } + }() + + err = client.Client.Create(context.Background(), node1) + Expect(err).ToNot(HaveOccurred()) + + err = client.Client.Create(context.Background(), node2) + Expect(err).ToNot(HaveOccurred()) + + err = speaker.RegisteSpeaker(eip.GetSpeakerName(), bgpFakeSpeak) + Expect(err).ToNot(HaveOccurred()) + err = client.Client.Create(context.Background(), eip) + Expect(err).ToNot(HaveOccurred()) + + err = speaker.RegisteSpeaker(eipLayer2.GetSpeakerName(), layer2FakeSpeak) + Expect(err).ToNot(HaveOccurred()) + err = client.Client.Create(context.Background(), eipLayer2) + Expect(err).ToNot(HaveOccurred()) + + // wait other controller, like eip + time.Sleep(1 * time.Second) + + close(done) +}, 60) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + close(stopCh) + err := testEnv.Stop() + Expect(err).ToNot(HaveOccurred()) +}) + +func checkEipUsage(eip *networkv1alpha2.Eip, usage int) func() bool { + return func() bool { + clone := eip.DeepCopy() + client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: eip.Namespace, + Name: eip.Name, + }, clone) + + if clone.Status.Usage == usage { + return true + } + return false + } +} + +func checkSvc(svc *corev1.Service, fn func(dst *corev1.Service) bool) func() bool { + return func() bool { + clone := svc.DeepCopy() + + client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: clone.Namespace, + Name: clone.Name, + }, clone) + + if len(clone.Status.LoadBalancer.Ingress) <= 0 { + return false + } + + return fn(clone) + } +} + +var _ = Describe("Porter LoadBalancer Service", func() { + BeforeEach(func() { + Eventually(checkEipUsage(eip, 0), 3*time.Second).Should(BeTrue()) + Eventually(checkEipUsage(eipLayer2, 0), 3*time.Second).Should(BeTrue()) + + cloneEP := endpoints.DeepCopy() + err := client.Client.Create(context.Background(), cloneEP) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + return client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: cloneEP.Namespace, + Name: cloneEP.Name, + }, cloneEP) + }, 3*time.Second).ShouldNot(HaveOccurred()) + + clone := svc.DeepCopy() + err = client.Client.Create(context.Background(), clone) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() error { + return client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: clone.Namespace, + Name: clone.Name, + }, clone) + }, 3*time.Second).ShouldNot(HaveOccurred()) + + Eventually(checkEipUsage(eip, 1), 3*time.Second).Should(BeTrue()) + Eventually(checkEipUsage(eipLayer2, 0), 3*time.Second).Should(BeTrue()) + }) + + AfterEach(func() { + // Why does deleting a service delete the endpoint, but + //creating a service does not create the endpoint. + clone := svc.DeepCopy() + err := client.Client.Delete(context.Background(), clone) + Expect(err).ToNot(HaveOccurred()) + Eventually(func() bool { + err = client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: clone.Namespace, + Name: clone.Name, + }, clone) + return k8serrors.IsNotFound(err) + }, 3*time.Second).Should(Equal(true)) + + cloneEP := endpoints.DeepCopy() + Eventually(func() bool { + err := client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: cloneEP.Namespace, + Name: cloneEP.Name, + }, cloneEP) + return k8serrors.IsNotFound(err) + }, 3*time.Second).Should(Equal(true)) + + Eventually(checkEipUsage(eip, 0), 3*time.Second).Should(BeTrue()) + Eventually(checkEipUsage(eipLayer2, 0), 3*time.Second).Should(BeTrue()) + }) + + It("Should add FinalizerName", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return util.ContainsString(dst.Finalizers, constant.FinalizerName) + }), 3*time.Second).Should(Equal(true)) + }) + + It("LoadBalancer Service should assign ingress ip", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return bgpFakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, + []string{ + node2.Status.Addresses[0].Address, + node1.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + + It("Nexthops should be all nodes", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return bgpFakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, + []string{ + node2.Status.Addresses[0].Address, + node1.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + + When("Endpoint is empty", func() { + BeforeEach(func() { + updateEndpoints(endpoints, func(dst *corev1.Endpoints) { + dst.Subsets = []corev1.EndpointSubset{ + { + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: node1.Status.Addresses[0].Address, + NodeName: &node1.Name, + }, + }, + Ports: []corev1.EndpointPort{ + { + Port: 80, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + }) + }) + It("the nexthops should be empty", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return bgpFakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, nil) + }), 3*time.Second).Should(Equal(true)) + }) + }) + + Context("ExternalTrafficPolicy == ServiceExternalTrafficPolicyTypeLocal", func() { + BeforeEach(func() { + updateSvc(svc, func(dst *corev1.Service) { + dst.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + }) + }) + + It("external local service should forward to loacl node", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return bgpFakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, + []string{ + node1.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + + It("nexthop should change when endpoint changed", func() { + updateEndpoints(endpoints, func(dst *corev1.Endpoints) { + dst.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: node2.Status.Addresses[0].Address, + NodeName: &node2.Name, + }, + }, + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: node1.Status.Addresses[0].Address, + NodeName: &node1.Name, + }, + }, + Ports: []corev1.EndpointPort{ + { + Port: 80, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + }) + + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return bgpFakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, + []string{ + node2.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + }) + + Context("Change to Layer2 LoadBalancer Service", func() { + BeforeEach(func() { + updateSvc(svc, func(dst *corev1.Service) { + dst.Annotations[constant.PorterProtocolAnnotationKey] = constant.PorterProtocolLayer2 + }) + }) + + It("Nexthops should not be all nodes", func() { + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + return !layer2FakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, + []string{ + node2.Status.Addresses[0].Address, + node1.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + + Context("ExternalTrafficPolicy == ServiceExternalTrafficPolicyTypeLocal", func() { + BeforeEach(func() { + updateSvc(svc, func(dst *corev1.Service) { + dst.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + }) + }) + + It("layer2 service should have annotation", func() { + Eventually(checkEipUsage(eip, 0), 3*time.Second).Should(BeTrue()) + Eventually(checkEipUsage(eipLayer2, 1), 3*time.Second).Should(BeTrue()) + + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + if dst.Annotations[constant.PorterLayer2Annotation] == node1.Status.Addresses[0].Address { + return true + } + + return layer2FakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, []string{ + node1.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + + By("When the endpoint changes, the annotation changes at the same time.") + updateEndpoints(endpoints, func(dst *corev1.Endpoints) { + dst.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: node2.Status.Addresses[0].Address, + NodeName: &node2.Name, + }, + }, + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: node1.Status.Addresses[0].Address, + NodeName: &node1.Name, + }, + }, + Ports: []corev1.EndpointPort{ + { + Port: 80, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + }) + + Eventually(checkSvc(svc, func(dst *corev1.Service) bool { + if dst.Annotations[constant.PorterLayer2Annotation] == node2.Status.Addresses[0].Address { + return true + } + + return layer2FakeSpeak.Equal(dst.Status.LoadBalancer.Ingress[0].IP, []string{ + node2.Status.Addresses[0].Address, + }) + }), 3*time.Second).Should(Equal(true)) + }) + }) + }) +}) + +func updateSvc(origin *corev1.Service, fn func(dst *corev1.Service)) { + clone := origin.DeepCopy() + + retry.RetryOnConflict(retry.DefaultBackoff, func() error { + client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: clone.Namespace, + Name: clone.Name, + }, clone) + fn(clone) + return client.Client.Update(context.Background(), clone) + }) +} + +func updateEndpoints(origin *corev1.Endpoints, fn func(dst *corev1.Endpoints)) { + clone := origin.DeepCopy() + + retry.RetryOnConflict(retry.DefaultBackoff, func() error { + err := client.Client.Get(context.Background(), types.NamespacedName{ + Namespace: clone.Namespace, + Name: clone.Name, + }, clone) + if err != nil { + return err + } + fn(clone) + return client.Client.Update(context.Background(), clone) + }) +} diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go deleted file mode 100644 index be92b971c..000000000 --- a/pkg/errors/errors.go +++ /dev/null @@ -1,35 +0,0 @@ -package errors - -const ( - UnknowError = iota - ParaInvalidError - EIPDuplicateError - EIPIsUsedError - EIPNotExist //assign or find - EIPNotEnoughError -) - -type PorterError struct { - Code int - Message string -} - -func (e PorterError) Error() string { - return e.Message -} - -func ReasonForError(err error) int { - switch t := err.(type) { - case PorterError: - return t.Code - } - return UnknowError -} - -func IsParaInvalidError(err error) bool { - if ReasonForError(err) == ParaInvalidError { - return true - } - - return false -} diff --git a/pkg/ipam/datastore.go b/pkg/ipam/datastore.go deleted file mode 100644 index ccf04b597..000000000 --- a/pkg/ipam/datastore.go +++ /dev/null @@ -1,344 +0,0 @@ -package ipam - -import ( - "fmt" - "github.com/kubesphere/porter/api/v1alpha1" - bgpserver "github.com/kubesphere/porter/pkg/bgp/serverd" - "github.com/kubesphere/porter/pkg/constant" - "github.com/kubesphere/porter/pkg/layer2" - "net" - "sync" - - "github.com/go-logr/logr" - "github.com/kubesphere/porter/pkg/errors" - "github.com/kubesphere/porter/pkg/util" - "github.com/mikioh/ipaddr" - "k8s.io/apimachinery/pkg/types" -) - -func NewDataStore(log logr.Logger, bgpServer bgpserver.AnnounceBgp) *DataStore { - return &DataStore{ - log: log.WithName("DataStore"), - IPPool: make(map[string]*CIDRResource), - responders: make(map[string]layer2.Responder), - bgpServer: bgpServer, - } -} - -type DataStore struct { - log logr.Logger - lock sync.Mutex - IPPool map[string]*CIDRResource - bgpServer bgpserver.AnnounceBgp - responders map[string]layer2.Responder -} - -//unsafe need locked -func (d *DataStore) isInteractOfCurrentPool(ipnets []*net.IPNet) bool { - - if ipnets == nil { - return false - } - - for _, cidr := range d.IPPool { - if cidr.IntersectsWith(ipnets) { - return true - } - } - return false -} - -type CIDRResource struct { - EIPRefName string - CIDRs []*net.IPNet - Used map[string]*EIPRef - Size int - UsingKnownIPs bool - Protocol string -} - -func (c *CIDRResource) IsFull() bool { - return len(c.Used) == c.Size -} - -func (c *CIDRResource) Contains(ip net.IP) bool { - if ip == nil { - return false - } - - for _, cidr := range c.CIDRs { - if cidr.Contains(ip) { - return true - } - } - return false -} - -func (c *CIDRResource) IntersectsWith(ipnets []*net.IPNet) bool { - if ipnets == nil { - return false - } - - for _, cidr := range c.CIDRs { - for _, ipnet := range ipnets { - if util.Intersect(cidr, ipnet) { - return true - } - } - } - return false -} - -type EIPRef struct { - EIPRefName string - Address string - Service types.NamespacedName -} - -type AssignIPResponse struct { - EIPRefName string - Address string -} - -func (d *DataStore) AddEIPPool(eip string, name string, usingKnownIPs bool, protocol string) error { - log := d.log.WithValues("CIDR", eip) - - if protocol == "" { - protocol = constant.PorterProtocolBGP - } - - d.lock.Lock() - defer d.lock.Unlock() - - ipnets, err := util.ParseAddress(eip) - if err != nil { - log.Info("eip is invalid") - return nil - } - - if len(ipnets) <= 0 { - log.Info("EIP range is invalid") - return nil - } - if _, ok := d.IPPool[name]; ok { - log.Info("Cannot add eips with same name", "name", name) - return nil - } - if d.isInteractOfCurrentPool(ipnets) { - log.Info("eip overlap") - return nil - } - - if protocol == constant.PorterProtocolLayer2 { - responder, err := layer2.NewResponder(ipaddr.NewCursor([]ipaddr.Prefix{*ipaddr.NewPrefix(ipnets[0])}).First().IP, d.log.WithName(name)) - if err != nil { - return err - } - d.responders[name] = responder - } - - d.IPPool[name] = &CIDRResource{ - EIPRefName: name, - CIDRs: ipnets, - Used: make(map[string]*EIPRef), - Size: util.GetValidAddressCount(eip), - UsingKnownIPs: usingKnownIPs, - Protocol: protocol, - } - log.Info("Added EIP to pool") - return nil -} - -func (d *DataStore) RemoveEIPPool(eip, name string) error { - d.lock.Lock() - defer d.lock.Unlock() - - log := d.log.WithValues("name", name, "eip", eip) - - if responder, ok := d.responders[name]; ok { - if err := responder.Close(); err != nil { - log.Error(err, "Cannot close responder") - return err - } - delete(d.responders, name) - } - - if res, ok := d.IPPool[name]; ok { - if len(res.Used) != 0 { - for key, val := range res.Used { - d.log.Info("Service is still using this pool", "service", val.Service, "ip", key) - } - log.Info("DataStore EIP inuse") - return errors.PorterError{Code: errors.EIPIsUsedError} - } - - delete(d.IPPool, name) - return nil - } - return nil -} - -func validateProtocol(protocol string) (string, error) { - switch protocol { - case constant.PorterProtocolBGP: - fallthrough - case constant.PorterProtocolLayer2: - return protocol, nil - case "": - return constant.PorterProtocolBGP, nil - default: - return "", errors.PorterError{Code: errors.ParaInvalidError} - } -} - -func (d *DataStore) AssignIP(serviceName, ns string, protocol string) (*AssignIPResponse, error) { - d.log.Info("Try to AssignIP to service", "Service", serviceName, "Namespace", ns) - - protocol, err := validateProtocol(protocol) - if err != nil { - return nil, err - } - - d.lock.Lock() - defer d.lock.Unlock() - - selectIP := &AssignIPResponse{} - for _, ips := range d.IPPool { - if ips.IsFull() || ips.Protocol != protocol { - continue - } - - for _, cidr := range ips.CIDRs { - c := ipaddr.NewCursor([]ipaddr.Prefix{*ipaddr.NewPrefix(cidr)}) - for pos := c.First(); pos != nil; pos = c.Next() { - ip := pos.IP - if !ips.UsingKnownIPs { - last := ip.To4()[3] - if last == 0 || last == 255 { - continue - } - } - if _, ok := ips.Used[ip.String()]; !ok { - ips.Used[ip.String()] = &EIPRef{ - EIPRefName: ips.EIPRefName, - Address: ip.String(), - Service: types.NamespacedName{Name: serviceName, Namespace: ns}, - } - selectIP.Address = ip.String() - selectIP.EIPRefName = ips.EIPRefName - d.log.Info("Assign IP to service", "Service", serviceName, "ip", selectIP.Address) - return selectIP, nil - } - } - } - } - return nil, errors.PorterError{Code: errors.EIPNotEnoughError} -} - -func (d *DataStore) AssignSpecifyIP(ipstr, protocol, serviceName, ns string) (*AssignIPResponse, error) { - protocol, err := validateProtocol(protocol) - if err != nil { - return nil, err - } - - d.lock.Lock() - defer d.lock.Unlock() - - ip := net.ParseIP(ipstr) - for _, ips := range d.IPPool { - if ips.Contains(ip) { - if !ips.UsingKnownIPs { - last := ip.To4()[3] - if last == 0 || last == 255 { - return nil, errors.PorterError{Code: errors.EIPNotExist} - } - } - if _, ok := ips.Used[ipstr]; ok { - return nil, errors.PorterError{Code: errors.EIPIsUsedError} - } - - ips.Used[ipstr] = &EIPRef{ - EIPRefName: ips.EIPRefName, - Address: ipstr, - Service: types.NamespacedName{Name: serviceName, Namespace: ns}, - } - return &AssignIPResponse{ - EIPRefName: ips.EIPRefName, - Address: ipstr, - }, nil - } - } - return nil, errors.PorterError{Code: errors.EIPNotExist} -} - -func (d *DataStore) UnassignIP(ipstr string) error { - d.lock.Lock() - defer d.lock.Unlock() - - ip := net.ParseIP(ipstr) - for _, ips := range d.IPPool { - if ips.Contains(ip) { - if _, ok := ips.Used[ipstr]; ok { - delete(ips.Used, ipstr) - return nil - } - return nil - } - } - return nil -} - -func (d *DataStore) GetPoolUsage(name string) (v1alpha1.EipStatus, error) { - d.lock.Lock() - defer d.lock.Unlock() - - status := v1alpha1.EipStatus{} - - pool, ok := d.IPPool[name] - if !ok { - return status, fmt.Errorf("EIP %s not exist", name) - } - - status.PoolSize = pool.Size - status.Usage = len(pool.Used) - if status.Usage == status.PoolSize { - status.Occupied = true - } else { - status.Occupied = false - } - - return status, nil -} - -type EIPStatus struct { - EIPRef *EIPRef - Used bool - Exist bool -} - -func (d *DataStore) GetEIPStatus(eip string) *EIPStatus { - d.lock.Lock() - defer d.lock.Unlock() - - ip := net.ParseIP(eip) - for _, ips := range d.IPPool { - if ips.Contains(ip) { - if ref, ok := ips.Used[eip]; ok { - return &EIPStatus{ - EIPRef: ref, - Used: true, - Exist: true, - } - } - return &EIPStatus{ - EIPRef: &EIPRef{ - EIPRefName: ips.EIPRefName, - Address: eip, - }, - Used: false, - Exist: true, - } - } - } - return &EIPStatus{} -} diff --git a/pkg/ipam/eip_status.go b/pkg/ipam/eip_status.go deleted file mode 100644 index 83e5790f2..000000000 --- a/pkg/ipam/eip_status.go +++ /dev/null @@ -1,67 +0,0 @@ -package ipam - -import ( - "context" - "reflect" - "time" - - "github.com/kubesphere/porter/api/v1alpha1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" -) - -const ( - DefaultSyncInterval = time.Second * 10 -) - -func (i *IPAM) Start(stop <-chan struct{}) error { - i.log.Info("Starting EIP Updater") - for { - select { - case <-stop: - i.log.Info("Recieve stop signal, stopping") - return nil - case <-time.After(i.syncInterval): - i.do() - } - } -} - -func (i *IPAM) do() { - eiplist := &v1alpha1.EipList{} - err := i.client.List(context.TODO(), eiplist) - if err != nil { - i.log.Error(err, "Failed to list eip, waiting for next try") - return - } - for _, eip := range eiplist.Items { - err = i.syncEIP(&eip) - if err != nil { - i.log.Error(err, "Failed to sync eip, waiting for next try", "Name", eip.Name, "CIDR", eip.Spec.Address) - } - } -} - -func (i *IPAM) syncEIP(eip *v1alpha1.Eip) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - original := &v1alpha1.Eip{} - err := i.client.Get(context.TODO(), types.NamespacedName{Name: eip.Name}, original) - if err != nil { - return err - } - if !original.ObjectMeta.DeletionTimestamp.IsZero() { - i.log.V(2).Info("eip is deleting, skipping", "eip", eip.Spec.Address) - return nil - } - status, err := i.ds.GetPoolUsage(eip.Name) - if err != nil { - return err - } - if reflect.DeepEqual(original.Status, status) { - return nil - } - original.Status = status - i.log.Info("current eip usage", "use", status.Usage, "total", status.PoolSize, "eip", original.Spec.Address) - return i.client.Status().Update(context.TODO(), original) - }) -} diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go deleted file mode 100644 index 956bc4ff4..000000000 --- a/pkg/ipam/ipam.go +++ /dev/null @@ -1,115 +0,0 @@ -package ipam - -import ( - "context" - "time" - - "github.com/go-logr/logr" - networkv1alpha1 "github.com/kubesphere/porter/api/v1alpha1" - "github.com/kubesphere/porter/pkg/constant" - "github.com/kubesphere/porter/pkg/util" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" -) - -type IPAM struct { - client client.Client - log logr.Logger - - ds *DataStore - - syncInterval time.Duration -} - -func NewIPAM(log logr.Logger, ds *DataStore) *IPAM { - return &IPAM{ - log: log, - ds: ds, - syncInterval: DefaultSyncInterval, - } -} - -func (i *IPAM) SetupWithManager(mgr manager.Manager) error { - i.client = mgr.GetClient() - - i.log.Info("Setting up EIPUpdater") - if err := mgr.Add(i); err != nil { - i.log.Error(nil, "Failed to run EIPUpdater") - return err - } - - return ctrl.NewControllerManagedBy(mgr).Named("IPAM"). - For(&networkv1alpha1.Eip{}). - WithEventFilter(predicate.Funcs{ - UpdateFunc: func(e event.UpdateEvent) bool { - //only support create and delete, because change EIP will affect a lot - //so we should make sure no service use EIP, and then delete , create a new one. - if !e.MetaNew.GetDeletionTimestamp().IsZero() { - return true - } - return false - }, - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - }). - Complete(i) -} - -func (i *IPAM) Reconcile(req ctrl.Request) (ctrl.Result, error) { - eip := &networkv1alpha1.Eip{} - err := i.client.Get(context.TODO(), req.NamespacedName, eip) - if err != nil { - if k8serrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - deleted, err := i.useFinalizerIfNeeded(eip) - if err != nil { - return ctrl.Result{}, err - } - if deleted { - return ctrl.Result{}, nil - } - - err = i.ds.AddEIPPool(eip.Spec.Address, eip.Name, eip.Spec.UsingKnownIPs, eip.Spec.Protocol) - return ctrl.Result{RequeueAfter: time.Second * 60}, err -} - -func (i *IPAM) useFinalizerIfNeeded(eip *networkv1alpha1.Eip) (bool, error) { - i.log.Info("handling finalizer") - if eip.ObjectMeta.DeletionTimestamp.IsZero() { - if !util.ContainsString(eip.ObjectMeta.Finalizers, constant.IPAMFinalizerName) { - eip.ObjectMeta.Finalizers = append(eip.ObjectMeta.Finalizers, constant.IPAMFinalizerName) - if err := i.client.Update(context.Background(), eip); err != nil { - return false, err - } - i.log.Info("Append Finalizer to eip") - return false, nil - } - } else { - // The object is being deleted - if util.ContainsString(eip.ObjectMeta.Finalizers, constant.IPAMFinalizerName) { - i.log.Info("Begin to remove finalizer") - if err := i.ds.RemoveEIPPool(eip.Spec.Address, eip.Name); err != nil { - return false, err - } - eip.ObjectMeta.Finalizers = util.RemoveString(eip.ObjectMeta.Finalizers, constant.IPAMFinalizerName) - if err := i.client.Update(context.Background(), eip); err != nil { - if k8serrors.IsNotFound(err) { - return true, nil - } - return false, err - } - i.log.Info("Remove Finalizer before eip deleted") - } - return true, nil - } - return false, nil -} diff --git a/pkg/ipam/ipam_suite_test.go b/pkg/ipam/ipam_suite_test.go deleted file mode 100644 index ff735e3fc..000000000 --- a/pkg/ipam/ipam_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package ipam_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestIpam(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Ipam Suite") -} diff --git a/pkg/ipam/ipam_test.go b/pkg/ipam/ipam_test.go deleted file mode 100644 index 2a879b2e8..000000000 --- a/pkg/ipam/ipam_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package ipam_test - -import ( - "net" - - "github.com/kubesphere/porter/pkg/constant" - - "github.com/kubesphere/porter/pkg/errors" - "github.com/kubesphere/porter/pkg/ipam" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" -) - -var ( - testIPNet *net.IPNet - testIPNetStr string -) - -var _ = Describe("Ipam", func() { - BeforeEach(func() { - testIPNetStr = "192.168.1.0/24" - _, testIPNet, _ = net.ParseCIDR(testIPNetStr) - }) - - It("Should be ok with single ip", func() { - ds := ipam.NewDataStore(ctrl.Log.WithName("setup"), nil) - singleIP := "1.1.1.1" - Expect(ds.AddEIPPool(singleIP, "singleEIP", false, constant.PorterProtocolBGP)).ShouldNot(HaveOccurred()) - Expect(*ds.GetEIPStatus(singleIP)).To(Equal(ipam.EIPStatus{ - Exist: true, - EIPRef: &ipam.EIPRef{ - EIPRefName: "singleEIP", - Address: singleIP, - }, - })) - ip, err := ds.AssignIP("testSVC", "test", constant.PorterProtocolBGP) - Expect(err).ShouldNot(HaveOccurred()) - Expect(ip.Address).To(Equal(singleIP)) - Expect(ds.UnassignIP(ip.Address)).ShouldNot(HaveOccurred()) - Expect(ds.RemoveEIPPool(singleIP, "singleEIP")).ShouldNot(HaveOccurred()) - }) - - It("Should be ok to add eip", func() { - ds := ipam.NewDataStore(ctrl.Log.WithName("setup"), nil) - - Expect(ds.AddEIPPool(testIPNetStr, "defaultPool", false, constant.PorterProtocolBGP)).ShouldNot(HaveOccurred()) - - Expect(ds.GetEIPStatus("192.168.2.1").Exist).To(BeFalse()) - Expect(*ds.GetEIPStatus("192.168.1.1")).To(Equal(ipam.EIPStatus{ - Exist: true, - EIPRef: &ipam.EIPRef{ - EIPRefName: "defaultPool", - Address: "192.168.1.1", - }, - })) - - ip, err := ds.AssignIP("testSVC", "test", constant.PorterProtocolBGP) - Expect(err).ShouldNot(HaveOccurred()) - Expect(testIPNet.Contains(net.ParseIP(ip.Address))).To(BeTrue()) - Expect(*ds.GetEIPStatus(ip.Address)).To(Equal(ipam.EIPStatus{ - Exist: true, - EIPRef: &ipam.EIPRef{ - EIPRefName: ip.EIPRefName, - Address: ip.Address, - Service: types.NamespacedName{ - Namespace: "test", - Name: "testSVC", - }, - }, - Used: true, - })) - _, err = ds.AssignSpecifyIP(ip.Address, constant.PorterProtocolBGP, "testSvc1", "default") - Expect(errors.ReasonForError(err)).Should(Equal(errors.EIPIsUsedError)) - - _, err = ds.AssignSpecifyIP("192.168.1.2", constant.PorterProtocolBGP, "testSvc1", "default") - Expect(err).ShouldNot(HaveOccurred()) - - _, err = ds.AssignSpecifyIP("192.168.1.0", constant.PorterProtocolBGP, "testSvc1", "default") - Expect(errors.ReasonForError(err)).Should(Equal(errors.EIPNotExist)) - - Expect(errors.ReasonForError(ds.RemoveEIPPool(testIPNetStr, "defaultPool"))).Should(Equal(errors.EIPIsUsedError)) - Expect(ds.UnassignIP(ip.Address)).ShouldNot(HaveOccurred()) - Expect(ds.UnassignIP("192.168.1.2")).ShouldNot(HaveOccurred()) - Expect(ds.RemoveEIPPool(testIPNetStr, "defaultPool")).ShouldNot(HaveOccurred()) - - Expect(ds.AddEIPPool("192.168.98.1", "oneeip", false, constant.PorterProtocolBGP)).ShouldNot(HaveOccurred()) - - ip, err = ds.AssignIP("testSVC", "test", constant.PorterProtocolBGP) - Expect(err).ShouldNot(HaveOccurred()) - Expect(ip.Address).To(Equal("192.168.98.1")) - - _, err = ds.AssignIP("testSVC", "test", constant.PorterProtocolBGP) - Expect(errors.ReasonForError(err)).Should(Equal(errors.EIPNotEnoughError)) - }) - - It("Should be ok to add two eip pool in the meantime", func() { - ds := ipam.NewDataStore(ctrl.Log.WithName("setup"), nil) - Expect(ds.AddEIPPool(testIPNetStr, "defaultPool", false, constant.PorterProtocolBGP)).ShouldNot(HaveOccurred()) - Expect(ds.AddEIPPool("192.168.2.2", "defaultPool1", false, constant.PorterProtocolBGP)).ShouldNot(HaveOccurred()) - resp, err := ds.AssignIP("testSvc1", "default", constant.PorterProtocolBGP) - Expect(err).ShouldNot(HaveOccurred()) - Expect(ds.UnassignIP(resp.Address)).ShouldNot(HaveOccurred()) - Expect(ds.RemoveEIPPool(testIPNetStr, "defaultPool")).ShouldNot(HaveOccurred()) - Expect(ds.RemoveEIPPool("192.168.2.2", "defaultPool1")).ShouldNot(HaveOccurred()) - }) -}) diff --git a/pkg/ipam/protocol.go b/pkg/ipam/protocol.go deleted file mode 100644 index 5571faa85..000000000 --- a/pkg/ipam/protocol.go +++ /dev/null @@ -1,55 +0,0 @@ -package ipam - -import ( - "net" - - "github.com/kubesphere/porter/pkg/constant" - portererror "github.com/kubesphere/porter/pkg/errors" -) - -func (d *DataStore) protocolForEIP(eip string) (string, string) { - d.lock.Lock() - defer d.lock.Unlock() - for _, p := range d.IPPool { - if p.Contains(net.ParseIP(eip)) { - return p.EIPRefName, p.Protocol - } - } - return "", "" -} - -func (d *DataStore) SetBalancer(ip string, nexthops []string) error { - - name, protocol := d.protocolForEIP(ip) - switch protocol { - case constant.PorterProtocolBGP: - return d.bgpServer.SetBalancer(ip, nexthops) - case constant.PorterProtocolLayer2: - if len(nexthops) <= 0 { - return d.DelBalancer(ip) - } - if d.responders[name] != nil { - return d.responders[name].Gratuitous(net.ParseIP(ip), net.ParseIP(nexthops[0])) - } - default: - return portererror.PorterError{Code: portererror.ParaInvalidError} - } - - return nil -} - -func (d *DataStore) DelBalancer(ip string) error { - name, protocol := d.protocolForEIP(ip) - switch protocol { - case constant.PorterProtocolBGP: - return d.bgpServer.DelBalancer(ip) - case constant.PorterProtocolLayer2: - if d.responders[name] != nil { - d.responders[name].DeleteIP(ip) - } - default: - return portererror.PorterError{Code: portererror.ParaInvalidError} - } - - return nil -} diff --git a/pkg/kubeutil/kubeutil.go b/pkg/kubeutil/kubeutil.go deleted file mode 100644 index da0e71b01..000000000 --- a/pkg/kubeutil/kubeutil.go +++ /dev/null @@ -1,49 +0,0 @@ -package kubeutil - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func GetServiceNodesIP(c client.Client, serv *corev1.Service) ([]string, error) { - endpoints := &corev1.Endpoints{} - err := c.Get(context.TODO(), types.NamespacedName{Namespace: serv.GetNamespace(), Name: serv.GetName()}, endpoints) - if err != nil { - return nil, err - } - if len(endpoints.Subsets) == 0 { - return nil, nil - } - nodes := make(map[string]bool) - for _, addr := range endpoints.Subsets[0].Addresses { - nodes[*addr.NodeName] = true - } - for _, addr := range endpoints.Subsets[0].NotReadyAddresses { - nodes[*addr.NodeName] = true - } - nodeIPMap, err := GetNodeIPMap(c) - if err != nil { - return nil, err - } - result := make([]string, 0) - for key := range nodes { - result = append(result, nodeIPMap[key]) - } - return result, nil -} - -func GetNodeIPMap(c client.Client) (map[string]string, error) { - nodeList := &corev1.NodeList{} - err := c.List(context.TODO(), nodeList) - if err != nil { - return nil, err - } - result := make(map[string]string) - for _, node := range nodeList.Items { - result[node.Name] = node.Status.Addresses[0].Address - } - return result, nil -} diff --git a/pkg/layer2/arp.go b/pkg/layer2/arp.go deleted file mode 100644 index 4cb5c8990..000000000 --- a/pkg/layer2/arp.go +++ /dev/null @@ -1,192 +0,0 @@ -package layer2 - -import ( - "fmt" - "github.com/go-logr/logr" - "github.com/j-keck/arping" - "github.com/mdlayher/arp" - "github.com/mdlayher/ethernet" - "github.com/mdlayher/raw" - "github.com/vishvananda/netlink" - "io" - "net" -) - -const protocolARP = 0x0806 - -type arpResponder struct { - logger logr.Logger - - intf *net.Interface - conn *arp.Client - p *raw.Conn - closed chan struct{} - - ip2mac map[string]*net.HardwareAddr -} - -func newARPResponder(log logr.Logger, ifi *net.Interface) (*arpResponder, error) { - p, err := raw.ListenPacket(ifi, protocolARP, nil) - if err != nil { - return nil, err - } - client, err := arp.New(ifi, p) - if err != nil { - return nil, fmt.Errorf("creating ARP responder for %q: %s", ifi.Name, err) - } - - ret := &arpResponder{ - logger: log.WithName("arpResponder"), - intf: ifi, - conn: client, - p: p, - closed: make(chan struct{}), - ip2mac: make(map[string]*net.HardwareAddr), - } - go ret.run() - return ret, nil -} - -func (a *arpResponder) Close() error { - close(a.closed) - return a.conn.Close() -} - -//The source mac address must be on the network card, otherwise arp spoof could drop you packets. -func generateArp(intfHW net.HardwareAddr, op arp.Operation, srcHW net.HardwareAddr, srcIP net.IP, dstHW net.HardwareAddr, dstIP net.IP) ([]byte, error) { - pkt, err := arp.NewPacket(op, srcHW, srcIP, dstHW, dstIP) - if err != nil { - return nil, err - } - - pb, err := pkt.MarshalBinary() - if err != nil { - return nil, err - } - - f := ðernet.Frame{ - Destination: dstHW, - Source: intfHW, - EtherType: ethernet.EtherTypeARP, - Payload: pb, - } - - fb, err := f.MarshalBinary() - if err != nil { - return nil, err - } - - return fb, err -} - -func (a *arpResponder) DeleteIP(ip string) { - delete(a.ip2mac, ip) -} - -func resolveIP(nodeIP net.IP, iface *net.Interface) (hwAddr net.HardwareAddr, err error) { - //Resolve mac - for i := 0; i < 3; i++ { - hwAddr, _, err = arping.PingOverIface(nodeIP, *iface) - if err != nil { - continue - } else { - break - } - } - - return -} - -func (a *arpResponder) Gratuitous(ip, nodeIP net.IP) error { - var ( - hwAddr net.HardwareAddr - err error - ) - - if ip.To4() == nil { - return nil - } - - routers, err := netlink.RouteGet(nodeIP) - if err != nil { - return err - } - - iface, err := net.InterfaceByIndex(routers[0].LinkIndex) - if err != nil { - return err - } - - if iface.Name != "lo" && routers[0].LinkIndex != a.intf.Index { - return nil - } - - if iface.Name == "lo" { - hwAddr = a.intf.HardwareAddr - } else { - hwAddr, err = resolveIP(nodeIP, a.intf) - if err != nil { - return err - } - } - - a.ip2mac[ip.String()] = &hwAddr - - for _, op := range []arp.Operation{arp.OperationRequest, arp.OperationReply} { - a.logger.Info("send gratuitous arp packet", "eip", ip, "nodeIP", nodeIP, "hwAddr", hwAddr) - - fb, err := generateArp(a.intf.HardwareAddr, op, hwAddr, ip, ethernet.Broadcast, ip) - if err != nil { - return err - } - - if _, err = a.p.WriteTo(fb, &raw.Addr{HardwareAddr: ethernet.Broadcast}); err != nil { - a.logger.Error(err, "send gratuitous arp packet") - return err - } - } - - return nil -} - -func (a *arpResponder) run() { - for a.processRequest() != dropReasonClosed { - } -} - -func (a *arpResponder) processRequest() dropReason { - pkt, _, err := a.conn.Read() - if err != nil { - // ARP listener doesn't cleanly return EOF when closed, so we - // need to hook into the call to arpResponder.close() - // independently. - select { - case <-a.closed: - return dropReasonClosed - default: - } - if err == io.EOF { - return dropReasonClosed - } - return dropReasonError - } - - // Ignore ARP replies. - if pkt.Operation != arp.OperationRequest { - return dropReasonARPReply - } - - hwAddr := a.ip2mac[pkt.TargetIP.String()] - if hwAddr == nil { - return dropReasonAnnounceIP - } - a.logger.Info("got ARP request, sending response", "interface", a.intf.Name, "ip", pkt.TargetIP, "senderIP", pkt.SenderIP, "senderMAC", pkt.SenderHardwareAddr, "responseMAC", hwAddr) - fb, err := generateArp(a.intf.HardwareAddr, arp.OperationReply, *hwAddr, pkt.TargetIP, pkt.SenderHardwareAddr, pkt.SenderIP) - if err != nil { - return dropReasonError - } - if _, err := a.p.WriteTo(fb, &raw.Addr{HardwareAddr: pkt.SenderHardwareAddr}); err != nil { - a.logger.Error(err, "op", "arpReply", "interface", a.intf.Name, "ip", pkt.TargetIP, "senderIP", pkt.SenderIP, "senderMAC", pkt.SenderHardwareAddr, "responseMAC", hwAddr) - } - return dropReasonNone -} diff --git a/pkg/layer2/arp_test.go b/pkg/layer2/arp_test.go deleted file mode 100644 index e78cbb68a..000000000 --- a/pkg/layer2/arp_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package layer2 - -import ( - "net" - - "github.com/mdlayher/arp" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var ( - testIntfHW, _ = net.ParseMAC("00:00:00:00:00:01") - testIntfHWStr = "00:00:00:00:00:01" - testIntfIP = net.ParseIP(testIntfIPStr[0]) - testIntfIPStr = []string{"127.0.0.1"} - testEIPHW, _ = net.ParseMAC("00:00:00:00:00:02") - testEIP = net.ParseIP("192.168.88.2") - testEIPStr = "192.168.88.2" - testDstHW, _ = net.ParseMAC("00:00:00:00:00:03") - testDstIP = net.ParseIP("192.168.88.3") - testDstIPStr = "192.168.88.3" -) - -var _ = Describe("generateArp", func() { - It("generateArp", func() { - _, err := generateArp(testIntfHW, arp.OperationReply, testEIPHW, testEIP, testDstHW, testDstIP) - Expect(err).ShouldNot(HaveOccurred()) - }) -}) diff --git a/pkg/layer2/doc.go b/pkg/layer2/doc.go deleted file mode 100644 index 06ef1e65f..000000000 --- a/pkg/layer2/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -package layer2 - -// Referenced part of the code in https://github.com/metallb/metallb/tree/main/internal/layer2 diff --git a/pkg/layer2/layer2_suite_test.go b/pkg/layer2/layer2_suite_test.go deleted file mode 100644 index b3f3644e7..000000000 --- a/pkg/layer2/layer2_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package layer2_test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestLayer2(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Layer2 Suite") -} diff --git a/pkg/layer2/responder.go b/pkg/layer2/responder.go deleted file mode 100644 index 163da42ba..000000000 --- a/pkg/layer2/responder.go +++ /dev/null @@ -1,55 +0,0 @@ -package layer2 - -import ( - "fmt" - "net" - - "github.com/go-logr/logr" - "github.com/vishvananda/netlink" -) - -type Responder interface { - DeleteIP(ip string) - Gratuitous(ip, nodeIP net.IP) error - Close() error -} - -func NewResponder(ip net.IP, log logr.Logger) (Responder, error) { - - routers, err := netlink.RouteGet(ip) - if err != nil { - return nil, err - } - - iface, err := net.InterfaceByIndex(routers[0].LinkIndex) - if err != nil { - return nil, err - } - - if ip.To4() != nil { - resp, err := newARPResponder(log, iface) - if err != nil { - return nil, err - } - - return resp, nil - } - - return nil, fmt.Errorf("Not vaild ip, only support ipv4 now") -} - -// dropReason is the reason why a layer2 protocol packet was not -// responded to. -type dropReason int - -// Various reasons why a packet was dropped. -const ( - dropReasonNone dropReason = iota - dropReasonClosed - dropReasonError - dropReasonARPReply - dropReasonMessageType - dropReasonNoSourceLL - dropReasonEthernetDestination - dropReasonAnnounceIP -) diff --git a/pkg/leader-elector/election.go b/pkg/leader-elector/election.go new file mode 100644 index 000000000..256476c70 --- /dev/null +++ b/pkg/leader-elector/election.go @@ -0,0 +1,62 @@ +package leader + +import ( + "context" + "github.com/kubesphere/porter/pkg/constant" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" + "os" + "time" +) + +var ( + Leader bool +) + +func envNamespace() string { + ns := os.Getenv(constant.EnvPorterNamespace) + if ns == "" { + return constant.PorterNamespace + } + return ns +} + +func envNodename() string { + name := os.Getenv(constant.EnvNodeName) + if name == "" { + panic("env NODE_NAME should not be empty") + } + return name +} + +func LeaderElector(client *clientset.Clientset) { + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: constant.PorterSpeakerLocker, + Namespace: envNamespace(), + }, + Client: client.CoordinationV1(), + LockConfig: resourcelock.ResourceLockConfig{ + Identity: envNodename(), + }, + } + + // start the leader election code loop + go leaderelection.RunOrDie(context.Background(), leaderelection.LeaderElectionConfig{ + Lock: lock, + ReleaseOnCancel: true, + LeaseDuration: 10 * time.Second, + RenewDeadline: 5 * time.Second, + RetryPeriod: 2 * time.Second, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(ctx context.Context) { + Leader = true + }, + OnStoppedLeading: func() { + Leader = false + }, + }, + }) +} diff --git a/pkg/leader-elector/options.go b/pkg/leader-elector/options.go new file mode 100644 index 000000000..c19daff39 --- /dev/null +++ b/pkg/leader-elector/options.go @@ -0,0 +1 @@ +package leader diff --git a/pkg/log/log.go b/pkg/log/log.go index ce6cb8c95..9b24b874c 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -1,30 +1,11 @@ package log import ( - "github.com/spf13/pflag" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) -type LogOptions struct { - *zap.Options -} - -func NewLogOptions() *LogOptions { - return &LogOptions{ - Options: &zap.Options{ - Development: false, - }, - } -} - -func (options *LogOptions) AddFlags(fs *pflag.FlagSet) { - // Set Development mode value - fs.BoolVar(&options.Development, "zap-devel", options.Development, - "Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn). "+ - "Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error)") -} - -func InitLog(options *LogOptions) { - ctrl.SetLogger(zap.New(zap.UseFlagOptions(options.Options))) +func InitLog(options *Options) { + log := zap.New(zap.UseFlagOptions(&options.Options)) + ctrl.SetLogger(log) } diff --git a/pkg/log/options.go b/pkg/log/options.go new file mode 100644 index 000000000..5b7b1b415 --- /dev/null +++ b/pkg/log/options.go @@ -0,0 +1,28 @@ +package log + +import ( + "flag" + "github.com/spf13/pflag" + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +type Options struct { + zap.Options +} + +func NewOptions() *Options { + return &Options{ + Options: zap.Options{ + Development: false, + Encoder: nil, + DestWritter: nil, + Level: zapcore.InfoLevel, + StacktraceLevel: nil, + ZapOpts: nil, + }} +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + o.BindFlags(flag.CommandLine) +} diff --git a/pkg/manager/client/client.go b/pkg/manager/client/client.go new file mode 100644 index 000000000..bd5e9f9dd --- /dev/null +++ b/pkg/manager/client/client.go @@ -0,0 +1,7 @@ +package client + +import "sigs.k8s.io/controller-runtime/pkg/client" + +var ( + Client client.Client +) diff --git a/pkg/manager/generic.go b/pkg/manager/generic.go new file mode 100644 index 000000000..91de2b62b --- /dev/null +++ b/pkg/manager/generic.go @@ -0,0 +1,55 @@ +package manager + +import ( + networkv1alpha2 "github.com/kubesphere/porter/api/v1alpha2" + "github.com/kubesphere/porter/pkg/manager/client" + "github.com/spf13/pflag" + admissionv1 "k8s.io/api/admission/v1" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" +) + +type GenericOptions struct { + MetricsAddr string + ReadinessAddr string +} + +func NewGenericOptions() *GenericOptions { + return &GenericOptions{ + MetricsAddr: ":8080", + ReadinessAddr: ":8000", + } +} + +func (options *GenericOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&options.MetricsAddr, "metrics-addr", options.MetricsAddr, "The address the metric endpoint binds to.") + fs.StringVar(&options.ReadinessAddr, "readiness-addr", options.ReadinessAddr, "The address readinessProbe used") +} + +func NewManager(cfg *rest.Config, options *GenericOptions) (ctrl.Manager, error) { + opts := ctrl.Options{ + Scheme: scheme, + } + if options != nil { + opts.MetricsBindAddress = options.MetricsAddr + } + result, err := ctrl.NewManager(cfg, opts) + + client.Client = result.GetClient() + + return result, err +} + +var ( + scheme = runtime.NewScheme() +) + +func init() { + _ = corev1.AddToScheme(scheme) + _ = admissionv1.AddToScheme(scheme) + _ = admissionv1beta1.AddToScheme(scheme) + _ = networkv1alpha2.AddToScheme(scheme) +} diff --git a/pkg/nettool/iptables.go b/pkg/nettool/iptables.go index 2137416c7..0521e752a 100644 --- a/pkg/nettool/iptables.go +++ b/pkg/nettool/iptables.go @@ -33,28 +33,4 @@ func DeletePortForwardOfBGP(iptableExec iptables.IptablesIface, routerIP, localI return iptableExec.Delete("nat", BgpNatChain, rule...) } -func OpenForwardForEIP(iptableExec iptables.IptablesIface, eip string) error { - ruleSpec := []string{"-d", eip, "-j", "ACCEPT"} - ok, err := iptableExec.Exists("filter", "FORWARD", ruleSpec...) - if err != nil { - return err - } - if ok { - return nil - } - return iptableExec.Append("filter", "FORWARD", ruleSpec...) -} - -func CloseForwardForEIP(iptableExec iptables.IptablesIface, eip string) error { - ruleSpec := []string{"-d", eip, "-j", "ACCEPT"} - ok, err := iptableExec.Exists("filter", "FORWARD", ruleSpec...) - if err != nil { - return err - } - if !ok { - return nil - } - return iptableExec.Delete("filter", "FORWARD", ruleSpec...) -} - const BgpNatChain = "PREROUTING-PORTER" diff --git a/pkg/speaker/bgp/bgp_test.go b/pkg/speaker/bgp/bgp_test.go new file mode 100644 index 000000000..25277b173 --- /dev/null +++ b/pkg/speaker/bgp/bgp_test.go @@ -0,0 +1,191 @@ +package bgp + +import ( + bgpapi "github.com/kubesphere/porter/api/v1alpha2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "testing" +) + +var ( + b *Bgp + ch chan struct{} +) + +func TestServerd(t *testing.T) { + RegisterFailHandler(Fail) + log := zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) + RunSpecs(t, "gobgpd Suite") +} + +var _ = BeforeSuite(func() { + By("Init bgp server and config") + bgpOptions := &BgpOptions{ + GrpcHosts: ":50052", + } + + b = NewGoBgpd(bgpOptions) + ch = make(chan struct{}) + + go b.Start(ch) +}) + +var _ = AfterSuite(func() { + By("stop bgp server") + close(ch) +}) + +var _ = Describe("BGP test", func() { + Context("Create/Update/Delete BgpConf", func() { + It("Add BgpConf", func() { + err := b.HandleBgpGlobalConfig(&bgpapi.BgpConf{ + Spec: bgpapi.BgpConfSpec{ + As: 65003, + RouterId: "10.0.255.254", + ListenPort: 17900, + }, + }, false) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Update BgpConf", func() { + err := b.HandleBgpGlobalConfig(&bgpapi.BgpConf{ + Spec: bgpapi.BgpConfSpec{ + As: 65002, + RouterId: "10.0.255.253", + ListenPort: 17901, + }, + }, false) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Delete BgpConf", func() { + err := b.HandleBgpGlobalConfig(&bgpapi.BgpConf{ + Spec: bgpapi.BgpConfSpec{ + RouterId: "10.0.255.254", + }, + }, true) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + Context("Create/Update/Delete BgpPeer", func() { + It("Add BgpPeer", func() { + Expect(b.HandleBgpPeer(&bgpapi.BgpPeer{ + Spec: bgpapi.BgpPeerSpec{ + Conf: &bgpapi.PeerConf{ + PeerAs: 65001, + NeighborAddress: "192.168.0.2", + }, + }, + }, false)).Should(HaveOccurred()) + }) + + It("Add BgpConf", func() { + err := b.HandleBgpGlobalConfig(&bgpapi.BgpConf{ + Spec: bgpapi.BgpConfSpec{ + As: 65003, + RouterId: "10.0.255.254", + ListenPort: 17900, + }, + }, false) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("Update BgpPeer", func() { + peer := &bgpapi.BgpPeer{ + Spec: bgpapi.BgpPeerSpec{ + Conf: &bgpapi.PeerConf{ + PeerAs: 65002, + NeighborAddress: "192.168.0.2", + }, + }, + } + Expect(b.HandleBgpPeer(peer, false)).ShouldNot(HaveOccurred()) + + peer.Spec.Conf.PeerAs = 65001 + Expect(b.HandleBgpPeer(peer, false)).ShouldNot(HaveOccurred()) + }) + + It("Delete BgpPeer", func() { + Expect(b.HandleBgpPeer(&bgpapi.BgpPeer{ + Spec: bgpapi.BgpPeerSpec{ + Conf: &bgpapi.PeerConf{ + NeighborAddress: "192.168.0.2", + }, + }, + }, true)).ShouldNot(HaveOccurred()) + }) + + Context("Reconcile Routes", func() { + It("Add BgpPeer", func() { + Expect(b.HandleBgpPeer(&bgpapi.BgpPeer{ + Spec: bgpapi.BgpPeerSpec{ + Conf: &bgpapi.PeerConf{ + PeerAs: 65001, + NeighborAddress: "192.168.0.2", + }, + }, + }, false)).ShouldNot(HaveOccurred()) + }) + + It("Should generate right number", func() { + a := generateIdentifier("192.168.98.1") + b := generateIdentifier("192.168.98.11") + c := generateIdentifier("192.168.98.133") + Expect(a).To(BeEquivalentTo(1)) + Expect(b).To(BeEquivalentTo(11)) + Expect(c).To(BeEquivalentTo(133)) + }) + + It("Should correctly add/delete all routes", func() { + ip := "100.100.100.100" + nexthops := []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} + + By("Init bgp should be empty") + err, toAdd, toDelete := b.retriveRoutes(ip, 32, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(toAdd)).Should(Equal(3)) + Expect(len(toDelete)).Should(Equal(0)) + + By("Add nexthops to bgp") + err = b.SetBalancer(ip, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + err, toAdd, toDelete = b.retriveRoutes(ip, 32, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(toAdd)).Should(Equal(0)) + Expect(len(toDelete)).Should(Equal(0)) + + By("Append a nexthop to bgp") + nexthops = append(nexthops, "4.4.4.4") + Expect(len(nexthops)).Should(Equal(4)) + err = b.SetBalancer(ip, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + err, toAdd, toDelete = b.retriveRoutes(ip, 32, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(toAdd)).Should(Equal(0)) + Expect(len(toDelete)).Should(Equal(0)) + + By("Delete two nexthops from bgp") + nexthops = nexthops[:len(nexthops)-2] + Expect(len(nexthops)).Should(Equal(2)) + err = b.SetBalancer(ip, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + err, toAdd, toDelete = b.retriveRoutes(ip, 32, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(toAdd)).Should(Equal(0)) + Expect(len(toDelete)).Should(Equal(0)) + + By("Delete all nexthops from bgp") + Expect(b.DelBalancer(ip)).ShouldNot(HaveOccurred()) + err, toAdd, toDelete = b.retriveRoutes(ip, 32, nexthops) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(toAdd)).Should(Equal(2)) + Expect(len(toDelete)).Should(Equal(0)) + }) + }) + }) +}) diff --git a/pkg/speaker/bgp/conf.go b/pkg/speaker/bgp/conf.go new file mode 100644 index 000000000..db38baefc --- /dev/null +++ b/pkg/speaker/bgp/conf.go @@ -0,0 +1,23 @@ +package bgp + +import ( + bgpapi "github.com/kubesphere/porter/api/v1alpha2" + api "github.com/osrg/gobgp/api" + "golang.org/x/net/context" +) + +func (b *Bgp) HandleBgpGlobalConfig(global *bgpapi.BgpConf, delete bool) error { + if delete { + return b.bgpServer.StopBgp(context.Background(), nil) + } + + request, err := global.Spec.ConverToGoBgpGlabalConf() + if err != nil { + return err + } + + b.bgpServer.StopBgp(context.Background(), nil) + return b.bgpServer.StartBgp(context.Background(), &api.StartBgpRequest{ + Global: request, + }) +} diff --git a/pkg/speaker/bgp/gobgpd.go b/pkg/speaker/bgp/gobgpd.go new file mode 100644 index 000000000..4e6a39076 --- /dev/null +++ b/pkg/speaker/bgp/gobgpd.go @@ -0,0 +1,85 @@ +package bgp + +import ( + "github.com/kubesphere/porter/pkg/speaker" + api "github.com/osrg/gobgp/api" + "github.com/osrg/gobgp/pkg/server" + "golang.org/x/net/context" + "google.golang.org/grpc" + ctrl "sigs.k8s.io/controller-runtime" + "sync" +) + +var _ speaker.Speaker = &Bgp{} + +func NewGoBgpd(bgpOptions *BgpOptions) *Bgp { + maxSize := 4 << 20 //4MB + grpcOpts := []grpc.ServerOption{grpc.MaxRecvMsgSize(maxSize), grpc.MaxSendMsgSize(maxSize)} + + bgpServer := server.NewBgpServer(server.GrpcListenAddress(bgpOptions.GrpcHosts), server.GrpcOption(grpcOpts)) + + return &Bgp{ + bgpServer: bgpServer, + log: ctrl.Log.WithName("bgpserver"), + } +} + +func (b *Bgp) run(stopCh <-chan struct{}) { + log := ctrl.Log.WithName("gobgpd") + + log.Info("gobgpd starting") + go b.bgpServer.Serve() + <-stopCh + log.Info("gobgpd ending") + err := b.bgpServer.StopBgp(context.Background(), &api.StopBgpRequest{}) + if err != nil { + log.Error(err, "failed to stop gobgpd") + } +} + +func (b *Bgp) Start(stopCh <-chan struct{}) error { + go b.run(stopCh) + return nil +} + +type cache struct { + lock sync.Mutex + confs map[string]interface{} +} + +var ( + confs *cache + peers *cache +) + +func (c *cache) set(k string, v interface{}) { + c.lock.Lock() + defer c.lock.Unlock() + + c.confs[k] = v +} + +func (c *cache) delete(k string) { + c.lock.Lock() + defer c.lock.Unlock() + + delete(c.confs, k) +} + +func (c *cache) get(k string) interface{} { + c.lock.Lock() + defer c.lock.Unlock() + + return c.confs[k] +} + +func init() { + confs = &cache{ + lock: sync.Mutex{}, + confs: make(map[string]interface{}), + } + peers = &cache{ + lock: sync.Mutex{}, + confs: make(map[string]interface{}), + } +} diff --git a/pkg/speaker/bgp/options.go b/pkg/speaker/bgp/options.go new file mode 100644 index 000000000..bc1d57a27 --- /dev/null +++ b/pkg/speaker/bgp/options.go @@ -0,0 +1,26 @@ +package bgp + +import ( + "github.com/go-logr/logr" + "github.com/osrg/gobgp/pkg/server" + "github.com/spf13/pflag" +) + +type BgpOptions struct { + GrpcHosts string `long:"api-hosts" description:"specify the hosts that gobgpd listens on" default:":50051"` +} + +func NewBgpOptions() *BgpOptions { + return &BgpOptions{ + GrpcHosts: ":50051", + } +} + +func (options *BgpOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&options.GrpcHosts, "api-hosts", options.GrpcHosts, "specify the hosts that gobgpd listens on") +} + +type Bgp struct { + bgpServer *server.BgpServer + log logr.Logger +} diff --git a/pkg/speaker/bgp/path.go b/pkg/speaker/bgp/path.go new file mode 100644 index 000000000..113d5e5cf --- /dev/null +++ b/pkg/speaker/bgp/path.go @@ -0,0 +1,215 @@ +package bgp + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + + "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/any" + api "github.com/osrg/gobgp/api" + bgppacket "github.com/osrg/gobgp/pkg/packet/bgp" +) + +func generateIdentifier(nexthop string) uint32 { + index := strings.LastIndex(nexthop, ".") + n, _ := strconv.ParseUint(nexthop[index+1:], 0, 32) + return uint32(n) +} + +func getFamily(ip string) *api.Family { + family := &api.Family{ + Afi: api.Family_AFI_IP, + Safi: api.Family_SAFI_UNICAST, + } + if net.ParseIP(ip).To4() == nil { + family = &api.Family{ + Afi: api.Family_AFI_IP6, + Safi: api.Family_SAFI_UNICAST, + } + } + + return family +} + +func toAPIPath(ip string, prefix uint32, nexthop string) *api.Path { + nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{ + Prefix: ip, + PrefixLen: prefix, + }) + a1, _ := ptypes.MarshalAny(&api.OriginAttribute{ + Origin: uint32(bgppacket.BGP_ORIGIN_ATTR_TYPE_IGP), + }) + a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{ + NextHop: nexthop, + }) + attrs := []*any.Any{a1, a2} + + return &api.Path{ + Family: getFamily(ip), + Nlri: nlri, + Pattrs: attrs, + Identifier: generateIdentifier(nexthop), + } +} + +func fromAPIPath(path *api.Path) net.IP { + for _, attr := range path.Pattrs { + var value ptypes.DynamicAny + + ptypes.UnmarshalAny(attr, &value) + + switch a := value.Message.(type) { + case *api.NextHopAttribute: + return net.ParseIP(a.NextHop) + } + } + + return nil +} + +func (b *Bgp) retriveRoutes(ip string, prefix uint32, nexthops []string) (err error, toAdd, toDelete []string) { + listPathRequest := &api.ListPathRequest{ + TableType: api.TableType_GLOBAL, + Family: getFamily(ip), + Prefixes: []*api.TableLookupPrefix{ + &api.TableLookupPrefix{ + Prefix: ip, + }, + }, + } + + origins := make(map[string]bool) + news := make(map[string]bool) + for _, item := range nexthops { + news[item] = true + } + found := false + fn := func(d *api.Destination) { + found = true + for _, path := range d.Paths { + nexthop := fromAPIPath(path) + origins[nexthop.String()] = true + } + //compare + for key := range origins { + if _, ok := news[key]; !ok { + toDelete = append(toDelete, key) + } + } + for key := range news { + if _, ok := origins[key]; !ok { + toAdd = append(toAdd, key) + } + } + } + + err = b.bgpServer.ListPath(context.Background(), listPathRequest, fn) + if err != nil { + return + } + if !found { + toAdd = nexthops + } + + return +} + +func (b *Bgp) ready() error { + response, err := b.bgpServer.GetBgp(context.Background(), nil) + if err != nil { + return err + } + + if response.Global.As == 0 { + return fmt.Errorf("Bgp not ready, please config bgpconf/bgppeer") + } + + return nil +} + +func (b *Bgp) SetBalancer(ip string, nexthops []string) error { + err := b.ready() + if err != nil { + return err + } + + prefix := uint32(32) + err, toAdd, toDelete := b.retriveRoutes(ip, prefix, nexthops) + if err != nil { + return err + } + + err = b.addMultiRoutes(ip, prefix, toAdd) + if err != nil { + return err + } + err = b.deleteMultiRoutes(ip, prefix, toDelete) + if err != nil { + return err + } + return nil +} + +func (b *Bgp) addMultiRoutes(ip string, prefix uint32, nexthops []string) error { + for _, nexthop := range nexthops { + apipath := toAPIPath(ip, prefix, nexthop) + _, err := b.bgpServer.AddPath(context.Background(), &api.AddPathRequest{ + Path: apipath, + }) + if err != nil { + return err + } + } + return nil +} + +func (b *Bgp) deleteMultiRoutes(ip string, prefix uint32, nexthops []string) error { + for _, nexthop := range nexthops { + apipath := toAPIPath(ip, prefix, nexthop) + err := b.bgpServer.DeletePath(context.Background(), &api.DeletePathRequest{ + Path: apipath, + }) + if err != nil { + return err + } + } + return nil +} + +func (b *Bgp) DelBalancer(ip string) error { + err := b.ready() + if err != nil { + return err + } + + lookup := &api.TableLookupPrefix{ + Prefix: ip, + } + listPathRequest := &api.ListPathRequest{ + TableType: api.TableType_GLOBAL, + Family: getFamily(ip), + Prefixes: []*api.TableLookupPrefix{lookup}, + } + var errDelete error + fn := func(d *api.Destination) { + for _, path := range d.Paths { + errDelete = b.bgpServer.DeletePath(context.Background(), &api.DeletePathRequest{ + Path: path, + }) + if errDelete != nil { + return + } + } + } + err = b.bgpServer.ListPath(context.Background(), listPathRequest, fn) + if err != nil { + return err + } + if errDelete != nil { + return errDelete + } + return nil +} diff --git a/pkg/speaker/bgp/peers.go b/pkg/speaker/bgp/peers.go new file mode 100644 index 000000000..91add27b9 --- /dev/null +++ b/pkg/speaker/bgp/peers.go @@ -0,0 +1,123 @@ +package bgp + +import ( + "fmt" + "net" + + bgpapi "github.com/kubesphere/porter/api/v1alpha2" + "github.com/kubesphere/porter/pkg/util" + api "github.com/osrg/gobgp/api" + "golang.org/x/net/context" + ctrl "sigs.k8s.io/controller-runtime" +) + +func defaultFamily(ip net.IP) *bgpapi.Family { + family := &bgpapi.Family{ + Afi: "AFI_IP", + Safi: "SAFI_UNICAST", + } + if ip.To4() == nil { + family = &bgpapi.Family{ + Afi: "AFI_IP6", + Safi: "SAFI_UNICAST", + } + } + + return family +} + +func (b *Bgp) HandleBgpPeerStatus(bgpPeers []bgpapi.BgpPeer) []*bgpapi.BgpPeer { + var ( + result []*bgpapi.BgpPeer + dels []*api.Peer + ) + + fn := func(peer *api.Peer) { + tmp, err := bgpapi.ConverStatusFromGoBgpPeer(peer) + if err != nil { + ctrl.Log.Error(err, "failed to ConverStatusFromGoBgpPeer", "peer", peer) + return + } + + var found *bgpapi.BgpPeer + + for _, bgpPeer := range bgpPeers { + if bgpPeer.Spec.Conf.NeighborAddress == tmp.PeerState.NeighborAddress { + found = &bgpPeer + break + } + } + + if found == nil { + dels = append(dels, peer) + } else { + clone := found.DeepCopy() + if clone.Status.NodesPeerStatus == nil { + clone.Status.NodesPeerStatus = make(map[string]bgpapi.NodePeerStatus) + } + + if clone.Spec.Conf.NeighborAddress == tmp.PeerState.NeighborAddress { + clone.Status.NodesPeerStatus[util.GetNodeName()] = tmp + } + + result = append(result, clone) + } + } + b.bgpServer.ListPeer(context.Background(), &api.ListPeerRequest{ + Address: "", + }, fn) + + for _, del := range dels { + ctrl.Log.Info("delete useless bgp peer", "peer", del) + b.bgpServer.DeletePeer(context.Background(), &api.DeletePeerRequest{ + Address: del.Conf.NeighborAddress, + Interface: del.Conf.NeighborInterface, + }) + } + + return result +} + +func (b *Bgp) HandleBgpPeer(neighbor *bgpapi.BgpPeer, delete bool) error { + // set default afisafi + if len(neighbor.Spec.AfiSafis) == 0 { + ip := net.ParseIP(neighbor.Spec.Conf.NeighborAddress) + if ip == nil { + return fmt.Errorf("field Spec.Conf.NeighborAddress invalid") + } + neighbor.Spec.AfiSafis = append(neighbor.Spec.AfiSafis, &bgpapi.AfiSafi{ + Config: &bgpapi.AfiSafiConfig{ + Family: defaultFamily(ip), + Enabled: true, + }, + AddPaths: &bgpapi.AddPaths{ + Config: &bgpapi.AddPathsConfig{ + SendMax: 10, + }, + }, + }) + } + + request, e := neighbor.Spec.ConverToGoBgpPeer() + if e != nil { + return e + } + + if delete { + b.bgpServer.DeletePeer(context.Background(), &api.DeletePeerRequest{ + Address: request.Conf.NeighborAddress, + Interface: request.Conf.NeighborInterface, + }) + } else { + _, e = b.bgpServer.UpdatePeer(context.Background(), &api.UpdatePeerRequest{ + Peer: request, + }) + if e != nil { + return b.bgpServer.AddPeer(context.Background(), &api.AddPeerRequest{ + Peer: request, + }) + } + } + + return nil +} diff --git a/pkg/speaker/layer2/arp.go b/pkg/speaker/layer2/arp.go new file mode 100644 index 000000000..f5d41e507 --- /dev/null +++ b/pkg/speaker/layer2/arp.go @@ -0,0 +1,267 @@ +package layer2 + +import ( + "fmt" + "github.com/kubesphere/porter/pkg/leader-elector" + "github.com/kubesphere/porter/pkg/speaker" + "io" + "net" + "reflect" + "sync" + + "github.com/go-logr/logr" + "github.com/j-keck/arping" + "github.com/mdlayher/arp" + "github.com/mdlayher/ethernet" + "github.com/mdlayher/raw" + "github.com/vishvananda/netlink" + ctrl "sigs.k8s.io/controller-runtime" +) + +const protocolARP = 0x0806 + +var _ speaker.Speaker = &arpSpeaker{} + +type arpSpeaker struct { + logger logr.Logger + + intf *net.Interface + conn *arp.Client + p *raw.Conn + + lock sync.Mutex + ip2mac map[string]net.HardwareAddr +} + +func (a *arpSpeaker) getMac(ip string) *net.HardwareAddr { + a.lock.Lock() + defer a.lock.Unlock() + + result, ok := a.ip2mac[ip] + if !ok { + return nil + } + return &result +} + +func (a *arpSpeaker) setMac(ip string, mac net.HardwareAddr) { + a.lock.Lock() + defer a.lock.Unlock() + + a.ip2mac[ip] = mac +} + +func newARPSpeaker(ifi *net.Interface) (*arpSpeaker, error) { + p, err := raw.ListenPacket(ifi, protocolARP, nil) + if err != nil { + return nil, err + } + client, err := arp.New(ifi, p) + if err != nil { + return nil, fmt.Errorf("creating ARP Speaker for %s, err=%v", ifi.Name, err) + } + + ret := &arpSpeaker{ + logger: ctrl.Log.WithName("arpSpeaker"), + intf: ifi, + conn: client, + p: p, + ip2mac: make(map[string]net.HardwareAddr), + } + + return ret, nil +} + +//The source mac address must be on the network card, otherwise arp spoof could drop you packets. +func generateArp(intfHW net.HardwareAddr, op arp.Operation, srcHW net.HardwareAddr, srcIP net.IP, dstHW net.HardwareAddr, dstIP net.IP) ([]byte, error) { + pkt, err := arp.NewPacket(op, srcHW, srcIP, dstHW, dstIP) + if err != nil { + return nil, err + } + + pb, err := pkt.MarshalBinary() + if err != nil { + return nil, err + } + + f := ðernet.Frame{ + Destination: dstHW, + Source: intfHW, + EtherType: ethernet.EtherTypeARP, + Payload: pb, + } + + fb, err := f.MarshalBinary() + if err != nil { + return nil, err + } + + return fb, err +} + +func (a *arpSpeaker) resolveIP(nodeIP net.IP) (hwAddr net.HardwareAddr, err error) { + routers, err := netlink.RouteGet(nodeIP) + if err != nil { + return nil, err + } + + iface, err := net.InterfaceByIndex(routers[0].LinkIndex) + if err != nil { + return nil, err + } + + if iface.Name == "lo" { + hwAddr = a.intf.HardwareAddr + } else { + //Resolve mac + for i := 0; i < 3; i++ { + hwAddr, _, err = arping.PingOverIface(nodeIP, *iface) + if err != nil { + continue + } else { + break + } + } + } + + if hwAddr != nil { + return hwAddr, nil + } + + return nil, err +} + +func (a *arpSpeaker) gratuitous(ip, nodeIP net.IP) error { + if !leader.Leader { + return nil + } + + hwAddr, err := a.resolveIP(nodeIP) + if err != nil { + return err + } + + tmp := a.getMac(ip.String()) + if tmp != nil && reflect.DeepEqual(hwAddr, *tmp) { + return nil + } + a.setMac(ip.String(), hwAddr) + + for _, op := range []arp.Operation{arp.OperationRequest, arp.OperationReply} { + a.logger.V(4).Info("send gratuitous arp packet", + "eip", ip, "nodeIP", nodeIP, "hwAddr", hwAddr) + + fb, err := generateArp(a.intf.HardwareAddr, op, hwAddr, ip, ethernet.Broadcast, ip) + if err != nil { + a.logger.Error(err, "generate gratuitous arp packet") + return err + } + + if _, err = a.p.WriteTo(fb, &raw.Addr{HardwareAddr: ethernet.Broadcast}); err != nil { + a.logger.Error(err, "send gratuitous arp packet") + return err + } + } + + return nil +} + +func (a *arpSpeaker) SetBalancer(ip string, nexthops []string) error { + return a.gratuitous(net.ParseIP(ip), net.ParseIP(nexthops[0])) +} + +func (a *arpSpeaker) DelBalancer(ip string) error { + a.lock.Lock() + defer a.lock.Unlock() + + delete(a.ip2mac, ip) + + return nil +} + +func (a *arpSpeaker) Start(stopCh <-chan struct{}) error { + go a.run(stopCh) + + go func() { + <-stopCh + a.conn.Close() + }() + + return nil +} + +func (a *arpSpeaker) run(stopCh <-chan struct{}) { + for { + err := a.processRequest() + + if err == dropReasonClosed { + return + } else if err == dropReasonError { + select { + case <-stopCh: + return + default: + } + } + } +} + +func (a *arpSpeaker) processRequest() dropReason { + pkt, _, err := a.conn.Read() + if err != nil { + if err == io.EOF { + return dropReasonClosed + } + + return dropReasonError + } + + if !leader.Leader { + return dropReasonLeader + } + + // Ignore ARP replies. + if pkt.Operation != arp.OperationRequest { + return dropReasonARPReply + } + + hwAddr := a.getMac(pkt.TargetIP.String()) + if hwAddr == nil { + return dropReasonUnknowTargetIP + } + + a.logger.V(4).Info("got ARP request, sending response", + "interface", a.intf.Name, + "ip", pkt.TargetIP, "senderIP", pkt.SenderIP, + "senderMAC", pkt.SenderHardwareAddr, "responseMAC", *hwAddr) + + fb, err := generateArp(a.intf.HardwareAddr, arp.OperationReply, *hwAddr, pkt.TargetIP, pkt.SenderHardwareAddr, pkt.SenderIP) + if err != nil { + a.logger.Error(err, "generate arp reply packet error") + return dropReasonError + } + + if _, err := a.p.WriteTo(fb, &raw.Addr{HardwareAddr: pkt.SenderHardwareAddr}); err != nil { + a.logger.Error(err, "send response", + "interface", a.intf.Name, + "ip", pkt.TargetIP, "senderIP", pkt.SenderIP, + "senderMAC", pkt.SenderHardwareAddr, "responseMAC", *hwAddr) + return dropReasonError + } + + return dropReasonNone +} + +// dropReason is the reason why a layer2 protocol packet was not +// responded to. +type dropReason int + +// Various reasons why a packet was dropped. +const ( + dropReasonNone dropReason = iota + dropReasonClosed + dropReasonError + dropReasonARPReply + dropReasonUnknowTargetIP + dropReasonLeader +) diff --git a/pkg/speaker/layer2/arp_test.go b/pkg/speaker/layer2/arp_test.go new file mode 100644 index 000000000..60f246d90 --- /dev/null +++ b/pkg/speaker/layer2/arp_test.go @@ -0,0 +1,140 @@ +package layer2 + +import ( + "net" + "os" + "testing" + "time" + + "github.com/j-keck/arping" + "github.com/kubesphere/porter/pkg/leader-elector" + "github.com/kubesphere/porter/pkg/speaker" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vishvananda/netlink" + "go.uber.org/zap/zapcore" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var ( + VethIfName = "testarp1" + VethIfIP = "192.168.166.1" + VethPeerIfName = "testarp2" + VethPeerIfIP = "192.168.166.2" + Gateway = "192.168.166.254" + Eip = "192.168.166.3" + veth = &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: VethIfName, + Flags: net.FlagUp, + MTU: 1500, + }, + PeerName: VethPeerIfName, + } +) + +func TestLayer2(t *testing.T) { + RegisterFailHandler(Fail) + if os.Getuid() != 0 { + //Skip("The test case requires root privileges.") + return + } + log := zap.New(zap.UseDevMode(true), zap.Level(zapcore.DebugLevel), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) + RunSpecs(t, "ARP Suite") +} + +var _ = BeforeSuite(func() { + leader.Leader = true + + err := netlink.LinkAdd(veth) + Expect(err).ShouldNot(HaveOccurred()) + + veth1, err := netlink.LinkByName(VethIfName) + Expect(err).ShouldNot(HaveOccurred()) + err = netlink.LinkSetUp(veth1) + Expect(err).ShouldNot(HaveOccurred()) + addr1, _ := netlink.ParseAddr("192.168.166.1/24") + err = netlink.AddrAdd(veth1, addr1) + Expect(err).ShouldNot(HaveOccurred()) + + veth2, err := netlink.LinkByName(VethPeerIfName) + Expect(err).ShouldNot(HaveOccurred()) + err = netlink.LinkSetUp(veth2) + Expect(err).ShouldNot(HaveOccurred()) + addr2, _ := netlink.ParseAddr("192.168.166.2/24") + err = netlink.AddrAdd(veth2, addr2) + Expect(err).ShouldNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + Expect(netlink.LinkDel(veth)).ShouldNot(HaveOccurred()) +}) + +var _ = Describe("new responder", func() { + Context("invalid interface string", func() { + It("specify invalid interface name", func() { + _, err := NewSpeaker(VethIfName+"tesfad", true) + Expect(err).Should(HaveOccurred()) + + _, err = NewSpeaker("haha:hahah", true) + Expect(err).Should(HaveOccurred()) + }) + It("specify can_reach, but ip invalid", func() { + _, err := NewSpeaker("can_reach:hahah", true) + Expect(err).Should(HaveOccurred()) + }) + It("specify can_reach, but interface is lo", func() { + _, err := NewSpeaker("can_reach:127.0.0.1", true) + Expect(err).Should(HaveOccurred()) + }) + }) + + Context("valid interface string", func() { + It("specify interface name", func() { + _, err := NewSpeaker(VethIfName, true) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("specify interface name", func() { + _, err := NewSpeaker("can_reach:192.168.166.254", true) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) +}) + +var _ = Describe("ARP Speak", func() { + var sp speaker.Speaker + + BeforeEach(func() { + var err error + sp, err = NewSpeaker(VethIfName, true) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(speaker.RegisteSpeaker(VethIfName, sp)).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + speaker.UnRegisteSpeaker(VethIfName) + }) + + It("set loadbalancer", func() { + err := sp.SetBalancer("192.168.166.3", []string{"192.168.166.1"}) + Expect(err).ShouldNot(HaveOccurred()) + peer, _ := net.InterfaceByName(VethPeerIfName) + mac, _, err := arping.PingOverIface(net.ParseIP("192.168.166.3"), *peer) + Expect(err).ShouldNot(HaveOccurred()) + veth, _ := net.InterfaceByName(VethIfName) + Expect(mac.String()).Should(Equal(veth.HardwareAddr.String())) + }) + + It("del loadbalancer", func() { + time.Sleep(10 * time.Second) + err := sp.DelBalancer("192.168.166.3") + Expect(err).ShouldNot(HaveOccurred()) + peer, _ := net.InterfaceByName(VethPeerIfName) + _, _, err = arping.PingOverIface(net.ParseIP("192.168.166.3"), *peer) + Expect(err).Should(HaveOccurred()) + }) +}) diff --git a/pkg/speaker/layer2/doc.go b/pkg/speaker/layer2/doc.go new file mode 100644 index 000000000..674630c39 --- /dev/null +++ b/pkg/speaker/layer2/doc.go @@ -0,0 +1 @@ +package layer2 diff --git a/pkg/speaker/layer2/responder.go b/pkg/speaker/layer2/responder.go new file mode 100644 index 000000000..8bafddb2e --- /dev/null +++ b/pkg/speaker/layer2/responder.go @@ -0,0 +1,63 @@ +package layer2 + +import ( + "fmt" + "net" + ctrl "sigs.k8s.io/controller-runtime" + "strings" + + "github.com/kubesphere/porter/pkg/speaker" + "github.com/vishvananda/netlink" +) + +func NewSpeaker(ifaceName string, v4 bool) (speaker.Speaker, error) { + var ( + iface *net.Interface + err error + ) + + strs := strings.SplitN(ifaceName, ":", 2) + if len(strs) == 1 { + iface, err = net.InterfaceByName(ifaceName) + if err != nil { + return nil, err + } + } else { + switch strs[0] { + case "can_reach": + ip := net.ParseIP(strs[1]) + if ip == nil { + return nil, fmt.Errorf("invalid can_reach address %s", strs[1]) + } + + routers, err := netlink.RouteGet(ip) + if err != nil { + return nil, err + } + + iface, err = net.InterfaceByIndex(routers[0].LinkIndex) + if err != nil { + return nil, err + } + + if iface.Name == "lo" { + return nil, fmt.Errorf("invalid interface lo") + } + default: + return nil, fmt.Errorf("invalid interface string, now only support can_reach") + } + } + + ctrl.Log.Info(fmt.Sprintf("use interface %s to speak arp", iface.Name)) + + if v4 { + speaker, err := newARPSpeaker(iface) + if err != nil { + return nil, err + } + + return speaker, nil + } + + return nil, fmt.Errorf("cannot create layer2 speaker, only support ipv4 now") +} diff --git a/pkg/speaker/speaker.go b/pkg/speaker/speaker.go new file mode 100644 index 000000000..ef78009a5 --- /dev/null +++ b/pkg/speaker/speaker.go @@ -0,0 +1,127 @@ +package speaker + +import ( + "github.com/projectcalico/libcalico-go/lib/set" + "sync" +) + +type Speaker interface { + SetBalancer(ip string, nexthops []string) error + DelBalancer(ip string) error + Start(stopCh <-chan struct{}) error +} + +type Fake struct { + lock sync.Mutex + nextHops map[string]set.Set +} + +func NewFake() *Fake { + return &Fake{ + nextHops: make(map[string]set.Set), + } +} + +func (f *Fake) SetBalancer(ip string, nexthops []string) error { + f.lock.Lock() + defer f.lock.Unlock() + + if len(nexthops) == 0 { + f.nextHops[ip] = nil + } else { + f.nextHops[ip] = set.FromArray(nexthops) + } + + return nil +} + +func (f *Fake) Equal(ip string, nexthops []string) bool { + f.lock.Lock() + defer f.lock.Unlock() + + tmp, ok := f.nextHops[ip] + if ok { + if tmp == nil && len(nexthops) == 0 { + return true + } + + if tmp == nil || len(nexthops) == 0 { + return false + } + } else { + if len(nexthops) == 0 { + return true + } + return false + } + + return set.FromArray(nexthops).Equals(f.nextHops[ip]) +} + +func (f *Fake) DelBalancer(ip string) error { + f.lock.Lock() + defer f.lock.Unlock() + + delete(f.nextHops, ip) + + return nil +} + +func (f *Fake) Start(stopCh <-chan struct{}) error { + return nil +} + +type speaker struct { + s Speaker + ch chan struct{} +} + +var ( + speakers map[string]speaker + lock sync.Mutex +) + +func RegisteSpeaker(name string, s Speaker) error { + lock.Lock() + defer lock.Unlock() + + t := speaker{ + s: s, + ch: make(chan struct{}), + } + + err := s.Start(t.ch) + if err == nil { + speakers[name] = t + return nil + } + + return err +} + +func UnRegisteSpeaker(name string) { + lock.Lock() + defer lock.Unlock() + + t, ok := speakers[name] + if ok { + close(t.ch) + } + delete(speakers, name) +} + +func GetSpeaker(name string) Speaker { + lock.Lock() + defer lock.Unlock() + + t, ok := speakers[name] + if ok { + return t.s + } + + return nil +} + +func init() { + speakers = make(map[string]speaker, 0) +} diff --git a/pkg/test/ip_test.go b/pkg/test/ip_test.go deleted file mode 100644 index 91cfc9061..000000000 --- a/pkg/test/ip_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package test - -import ( - "github.com/kubesphere/porter/pkg/util" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Util", func() { - Context("Tests of net function", func() { - It("Should get right ip", func() { - ip := util.GetOutboundIP() - Expect(ip).ShouldNot(BeZero()) - }) - It("Should print right string", func() { - str := util.ToCommonString("1.0.0.1", 24) - Expect(str).To(Equal("1.0.0.1/24")) - }) - }) -}) diff --git a/pkg/test/porter_suite_test.go b/pkg/test/porter_suite_test.go deleted file mode 100644 index 5cf06530b..000000000 --- a/pkg/test/porter_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package test - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestUtil(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Util suite") -} diff --git a/pkg/test/util.go b/pkg/test/util.go new file mode 100644 index 000000000..e60df221a --- /dev/null +++ b/pkg/test/util.go @@ -0,0 +1,51 @@ +package test + +import ( + "context" + "github.com/kubesphere/porter/pkg/manager/client" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "math/rand" + "time" +) + +// SetupTest will set up a testing environment. +func SetupTest(ctx context.Context) *core.Namespace { + var stopCh chan struct{} + ns := &core.Namespace{} + + BeforeEach(func() { + stopCh = make(chan struct{}) + *ns = core.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)}, + } + + err := client.Client.Create(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + }) + + AfterEach(func() { + close(stopCh) + + err := client.Client.Delete(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") + }) + + return ns +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} diff --git a/pkg/util/cidr/cidr.go b/pkg/util/cidr/cidr.go deleted file mode 100644 index 605ba3b00..000000000 --- a/pkg/util/cidr/cidr.go +++ /dev/null @@ -1,242 +0,0 @@ -package cidr - -import ( - "fmt" - "math/big" - "net" -) - -// Subnet takes a parent CIDR range and creates a subnet within it -// with the given number of additional prefix bits and the given -// network number. -// -// For example, 10.3.0.0/16, extended by 8 bits, with a network number -// of 5, becomes 10.3.5.0/24 . -func Subnet(base *net.IPNet, newBits int, num int) (*net.IPNet, error) { - ip := base.IP - mask := base.Mask - - parentLen, addrLen := mask.Size() - newPrefixLen := parentLen + newBits - - if newPrefixLen > addrLen { - return nil, fmt.Errorf("insufficient address space to extend prefix of %d by %d", parentLen, newBits) - } - - maxNetNum := uint64(1< maxNetNum { - return nil, fmt.Errorf("prefix extension of %d does not accommodate a subnet numbered %d", newBits, num) - } - - return &net.IPNet{ - IP: insertNumIntoIP(ip, big.NewInt(int64(num)), newPrefixLen), - Mask: net.CIDRMask(newPrefixLen, addrLen), - }, nil -} - -// Host takes a parent CIDR range and turns it into a host IP address with -// the given host number. -// -// For example, 10.3.0.0/16 with a host number of 2 gives 10.3.0.2. -func Host(base *net.IPNet, num int) (net.IP, error) { - ip := base.IP - mask := base.Mask - bigNum := big.NewInt(int64(num)) - - parentLen, addrLen := mask.Size() - hostLen := addrLen - parentLen - - maxHostNum := big.NewInt(int64(1)) - maxHostNum.Lsh(maxHostNum, uint(hostLen)) - maxHostNum.Sub(maxHostNum, big.NewInt(1)) - - numUint64 := big.NewInt(int64(bigNum.Uint64())) - if bigNum.Cmp(big.NewInt(0)) == -1 { - numUint64.Neg(bigNum) - numUint64.Sub(numUint64, big.NewInt(int64(1))) - bigNum.Sub(maxHostNum, numUint64) - } - - if numUint64.Cmp(maxHostNum) == 1 { - return nil, fmt.Errorf("prefix of %d does not accommodate a host numbered %d", parentLen, num) - } - var bitlength int - if ip.To4() != nil { - bitlength = 32 - } else { - bitlength = 128 - } - return insertNumIntoIP(ip, bigNum, bitlength), nil -} - -// AddressRange returns the first and last addresses in the given CIDR range. -func AddressRange(network *net.IPNet) (net.IP, net.IP) { - // the first IP is easy - firstIP := network.IP - - // the last IP is the network address OR NOT the mask address - prefixLen, bits := network.Mask.Size() - if prefixLen == bits { - // Easy! - // But make sure that our two slices are distinct, since they - // would be in all other cases. - lastIP := make([]byte, len(firstIP)) - copy(lastIP, firstIP) - return firstIP, lastIP - } - - firstIPInt, bits := ipToInt(firstIP) - hostLen := uint(bits) - uint(prefixLen) - lastIPInt := big.NewInt(1) - lastIPInt.Lsh(lastIPInt, hostLen) - lastIPInt.Sub(lastIPInt, big.NewInt(1)) - lastIPInt.Or(lastIPInt, firstIPInt) - - return firstIP, intToIP(lastIPInt, bits) -} - -// AddressCount returns the number of distinct host addresses within the given -// CIDR range. -// -// Since the result is a uint64, this function returns meaningful information -// only for IPv4 ranges and IPv6 ranges with a prefix size of at least 65. -func AddressCount(network *net.IPNet) uint64 { - prefixLen, bits := network.Mask.Size() - return 1 << (uint64(bits) - uint64(prefixLen)) -} - -//VerifyNoOverlap takes a list subnets and supernet (CIDRBlock) and verifies -//none of the subnets overlap and all subnets are in the supernet -//it returns an error if any of those conditions are not satisfied -func VerifyNoOverlap(subnets []*net.IPNet, CIDRBlock *net.IPNet) error { - firstLastIP := make([][]net.IP, len(subnets)) - for i, s := range subnets { - first, last := AddressRange(s) - firstLastIP[i] = []net.IP{first, last} - } - for i, s := range subnets { - if !CIDRBlock.Contains(firstLastIP[i][0]) || !CIDRBlock.Contains(firstLastIP[i][1]) { - return fmt.Errorf("%s does not fully contain %s", CIDRBlock.String(), s.String()) - } - for j := 0; j < len(subnets); j++ { - if i == j { - continue - } - - first := firstLastIP[j][0] - last := firstLastIP[j][1] - if s.Contains(first) || s.Contains(last) { - return fmt.Errorf("%s overlaps with %s", subnets[j].String(), s.String()) - } - } - } - return nil -} - -// PreviousSubnet returns the subnet of the desired mask in the IP space -// just lower than the start of IPNet provided. If the IP space rolls over -// then the second return value is true -func PreviousSubnet(network *net.IPNet, prefixLen int) (*net.IPNet, bool) { - startIP := checkIPv4(network.IP) - previousIP := make(net.IP, len(startIP)) - copy(previousIP, startIP) - cMask := net.CIDRMask(prefixLen, 8*len(previousIP)) - previousIP = Dec(previousIP) - previous := &net.IPNet{IP: previousIP.Mask(cMask), Mask: cMask} - if startIP.Equal(net.IPv4zero) || startIP.Equal(net.IPv6zero) { - return previous, true - } - return previous, false -} - -// NextSubnet returns the next available subnet of the desired mask size -// starting for the maximum IP of the offset subnet -// If the IP exceeds the maxium IP then the second return value is true -func NextSubnet(network *net.IPNet, prefixLen int) (*net.IPNet, bool) { - _, currentLast := AddressRange(network) - mask := net.CIDRMask(prefixLen, 8*len(currentLast)) - currentSubnet := &net.IPNet{IP: currentLast.Mask(mask), Mask: mask} - _, last := AddressRange(currentSubnet) - last = Inc(last) - next := &net.IPNet{IP: last.Mask(mask), Mask: mask} - if last.Equal(net.IPv4zero) || last.Equal(net.IPv6zero) { - return next, true - } - return next, false -} - -//Inc increases the IP by one this returns a new []byte for the IP -func Inc(IP net.IP) net.IP { - IP = checkIPv4(IP) - incIP := make([]byte, len(IP)) - copy(incIP, IP) - for j := len(incIP) - 1; j >= 0; j-- { - incIP[j]++ - if incIP[j] > 0 { - break - } - } - return incIP -} - -//Dec decreases the IP by one this returns a new []byte for the IP -func Dec(IP net.IP) net.IP { - IP = checkIPv4(IP) - decIP := make([]byte, len(IP)) - copy(decIP, IP) - decIP = checkIPv4(decIP) - for j := len(decIP) - 1; j >= 0; j-- { - decIP[j]-- - if decIP[j] < 255 { - break - } - } - return decIP -} - -func checkIPv4(ip net.IP) net.IP { - // Go for some reason allocs IPv6len for IPv4 so we have to correct it - if v4 := ip.To4(); v4 != nil { - return v4 - } - return ip -} - -func ipToInt(ip net.IP) (*big.Int, int) { - val := &big.Int{} - val.SetBytes([]byte(ip)) - if len(ip) == net.IPv4len { - return val, 32 - } else if len(ip) == net.IPv6len { - return val, 128 - } else { - panic(fmt.Errorf("Unsupported address length %d", len(ip))) - } -} - -func intToIP(ipInt *big.Int, bits int) net.IP { - ipBytes := ipInt.Bytes() - ret := make([]byte, bits/8) - // Pack our IP bytes into the end of the return array, - // since big.Int.Bytes() removes front zero padding. - for i := 1; i <= len(ipBytes); i++ { - ret[len(ret)-i] = ipBytes[len(ipBytes)-i] - } - return net.IP(ret) -} - -func insertNumIntoIP(ip net.IP, bigNum *big.Int, prefixLen int) net.IP { - ipInt, totalBits := ipToInt(ip) - bigNum.Lsh(bigNum, uint(totalBits-prefixLen)) - ipInt.Or(ipInt, bigNum) - return intToIP(ipInt, totalBits) -} - -func LoopForEachAddressInCIDR(ipnet *net.IPNet, f func(ip net.IP) bool) { - first, _ := AddressRange(ipnet) - for ; ipnet.Contains(first); first = Inc(first) { - if f(first) { - break - } - } -} diff --git a/pkg/util/cidr/cidr_suite_test.go b/pkg/util/cidr/cidr_suite_test.go deleted file mode 100644 index 35a77780d..000000000 --- a/pkg/util/cidr/cidr_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package cidr - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestCidr(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Cidr Suite") -} diff --git a/pkg/util/cidr/cidr_test.go b/pkg/util/cidr/cidr_test.go deleted file mode 100644 index 38a1df029..000000000 --- a/pkg/util/cidr/cidr_test.go +++ /dev/null @@ -1,334 +0,0 @@ -package cidr - -import ( - "bytes" - "net" - "strconv" -) - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Cidr", func() { - It("Should be ok to test Subnet", func() { - type Case struct { - Base string - Bits int - Num int - Output string - Error bool - } - - cases := []Case{ - Case{ - Base: "192.168.2.0/20", - Bits: 4, - Num: 6, - Output: "192.168.6.0/24", - }, - Case{ - Base: "192.168.2.0/20", - Bits: 4, - Num: 0, - Output: "192.168.0.0/24", - }, - Case{ - Base: "192.168.0.0/31", - Bits: 1, - Num: 1, - Output: "192.168.0.1/32", - }, - Case{ - Base: "192.168.0.0/21", - Bits: 4, - Num: 7, - Output: "192.168.3.128/25", - }, - Case{ - Base: "fe80::/48", - Bits: 16, - Num: 6, - Output: "fe80:0:0:6::/64", - }, - Case{ - Base: "fe80::/49", - Bits: 16, - Num: 7, - Output: "fe80:0:0:3:8000::/65", - }, - Case{ - Base: "192.168.2.0/31", - Bits: 2, - Num: 0, - Error: true, // not enough bits to expand into - }, - Case{ - Base: "fe80::/126", - Bits: 4, - Num: 0, - Error: true, // not enough bits to expand into - }, - Case{ - Base: "192.168.2.0/24", - Bits: 4, - Num: 16, - Error: true, // can't fit 16 into 4 bits - }, - } - - for _, testCase := range cases { - _, base, _ := net.ParseCIDR(testCase.Base) - gotNet, err := Subnet(base, testCase.Bits, testCase.Num) - if testCase.Error { - Expect(err).Should(HaveOccurred()) - } else { - Expect(err).ShouldNot(HaveOccurred()) - Expect(gotNet.String()).To(Equal(testCase.Output)) - } - } - }) - It("Should be ok to test Host", func() { - type Case struct { - Range string - Num int - Output string - Error bool - } - - cases := []Case{ - Case{ - Range: "192.168.2.0/20", - Num: 6, - Output: "192.168.0.6", - }, - Case{ - Range: "192.168.0.0/20", - Num: 257, - Output: "192.168.1.1", - }, - Case{ - Range: "2001:db8::/32", - Num: 1, - Output: "2001:db8::1", - }, - Case{ - Range: "192.168.1.0/24", - Num: 256, - Error: true, // only 0-255 will fit in 8 bits - }, - Case{ - Range: "192.168.0.0/30", - Num: -3, - Output: "192.168.0.1", // 4 address (0-3) in 2 bits; 3rd from end = 1 - }, - Case{ - Range: "192.168.0.0/30", - Num: -4, - Output: "192.168.0.0", // 4 address (0-3) in 2 bits; 4th from end = 0 - }, - Case{ - Range: "192.168.0.0/30", - Num: -5, - Error: true, // 4 address (0-3) in 2 bits; cannot accomodate 5 - }, - Case{ - Range: "fd9d:bc11:4020::/64", - Num: 2, - Output: "fd9d:bc11:4020::2", - }, - Case{ - Range: "fd9d:bc11:4020::/64", - Num: -2, - Output: "fd9d:bc11:4020:0:ffff:ffff:ffff:fffe", - }, - } - - for _, testCase := range cases { - _, network, _ := net.ParseCIDR(testCase.Range) - gotIP, err := Host(network, testCase.Num) - if testCase.Error { - Expect(err).Should(HaveOccurred()) - } else { - Expect(err).ShouldNot(HaveOccurred()) - Expect(gotIP.String()).To(Equal(testCase.Output)) - } - } - }) - - It("Should be ok to test AddressRange", func() { - type Case struct { - Range string - First string - Last string - } - - cases := []Case{ - Case{ - Range: "192.168.0.0/16", - First: "192.168.0.0", - Last: "192.168.255.255", - }, - Case{ - Range: "192.168.0.0/17", - First: "192.168.0.0", - Last: "192.168.127.255", - }, - Case{ - Range: "fe80::/64", - First: "fe80::", - Last: "fe80::ffff:ffff:ffff:ffff", - }, - } - - for _, testCase := range cases { - _, network, _ := net.ParseCIDR(testCase.Range) - firstIP, lastIP := AddressRange(network) - Expect(firstIP.String()).To(Equal(testCase.First)) - Expect(lastIP.String()).To(Equal(testCase.Last)) - } - }) - It("Should be ok to test IncDes", func() { - - testCase := [][]string{ - []string{"0.0.0.0", "0.0.0.1"}, - []string{"10.0.0.0", "10.0.0.1"}, - []string{"9.255.255.255", "10.0.0.0"}, - []string{"255.255.255.255", "0.0.0.0"}, - []string{"::", "::1"}, - []string{"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "::"}, - []string{"2001:db8:c001:ba00::", "2001:db8:c001:ba00::1"}, - } - - for _, tc := range testCase { - ip1 := net.ParseIP(tc[0]) - ip2 := net.ParseIP(tc[1]) - iIP := Inc(ip1) - Expect(iIP.Equal(ip2)).To(BeTrue()) - Expect(ip1.Equal(ip2)).To(BeFalse()) - } - for _, tc := range testCase { - ip1 := net.ParseIP(tc[0]) - ip2 := net.ParseIP(tc[1]) - dIP := Dec(ip2) - Expect(ip1.Equal(dIP)).To(BeTrue()) - Expect(ip2.Equal(dIP)).To(BeFalse()) - } - }) - It("Should be ok to test PreviousSubnet", func() { - - testCases := [][]string{ - []string{"10.0.0.0/24", "9.255.255.0/24", "false"}, - []string{"100.0.0.0/26", "99.255.255.192/26", "false"}, - []string{"0.0.0.0/26", "255.255.255.192/26", "true"}, - []string{"2001:db8:e000::/36", "2001:db8:d000::/36", "false"}, - []string{"::/64", "ffff:ffff:ffff:ffff::/64", "true"}, - } - for _, tc := range testCases { - _, c1, _ := net.ParseCIDR(tc[0]) - _, c2, _ := net.ParseCIDR(tc[1]) - mask, _ := c1.Mask.Size() - p1, rollback := PreviousSubnet(c1, mask) - Expect(p1.IP.Equal(c2.IP)).To(BeTrue(), "IP expected %v, got %v\n", c2.IP, p1.IP) - Expect(bytes.Equal(p1.Mask, c2.Mask)).To(BeTrue(), "Mask expected %v, got %v\n", c2.Mask, p1.Mask) - Expect(p1.String()).To(Equal(c2.String())) - check, _ := strconv.ParseBool(tc[2]) - Expect(rollback).To(Equal(check)) - } - for _, tc := range testCases { - _, c1, _ := net.ParseCIDR(tc[0]) - _, c2, _ := net.ParseCIDR(tc[1]) - mask, _ := c1.Mask.Size() - n1, rollover := NextSubnet(c2, mask) - Expect(n1.IP.Equal(c1.IP)).To(BeTrue(), "IP expected %v, got %v\n", c1.IP, n1.IP) - Expect(bytes.Equal(n1.Mask, c1.Mask)).To(BeTrue(), "Mask expected %v, got %v\n", c1.Mask, n1.Mask) - Expect(n1.String()).To(Equal(c1.String())) - check, _ := strconv.ParseBool(tc[2]) - Expect(rollover).To(Equal(check)) - } - }) - It("Should be ok to test VerifyNetowrk", func() { - - type testVerifyNetwork struct { - CIDRBlock string - CIDRList []string - } - - testCases := []*testVerifyNetwork{ - &testVerifyNetwork{ - CIDRBlock: "192.168.8.0/21", - CIDRList: []string{ - "192.168.8.0/24", - "192.168.9.0/24", - "192.168.10.0/24", - "192.168.11.0/25", - "192.168.11.128/25", - "192.168.12.0/25", - "192.168.12.128/26", - "192.168.12.192/26", - "192.168.13.0/26", - "192.168.13.64/27", - "192.168.13.96/27", - "192.168.13.128/27", - }, - }, - } - failCases := []*testVerifyNetwork{ - &testVerifyNetwork{ - CIDRBlock: "192.168.8.0/21", - CIDRList: []string{ - "192.168.8.0/24", - "192.168.9.0/24", - "192.168.10.0/24", - "192.168.11.0/25", - "192.168.11.128/25", - "192.168.12.0/25", - "192.168.12.64/26", - "192.168.12.128/26", - }, - }, - &testVerifyNetwork{ - CIDRBlock: "192.168.8.0/21", - CIDRList: []string{ - "192.168.7.0/24", - "192.168.9.0/24", - "192.168.10.0/24", - "192.168.11.0/25", - "192.168.11.128/25", - "192.168.12.0/25", - "192.168.12.64/26", - "192.168.12.128/26", - }, - }, - &testVerifyNetwork{ - CIDRBlock: "10.42.0.0/24", - CIDRList: []string{ - - "10.42.0.16/28", - "10.42.0.32/28", - "10.42.0.0/24", - }, - }, - } - - for _, tc := range testCases { - subnets := make([]*net.IPNet, len(tc.CIDRList)) - for i, s := range tc.CIDRList { - _, n, _ := net.ParseCIDR(s) - subnets[i] = n - } - _, CIDRBlock, _ := net.ParseCIDR(tc.CIDRBlock) - Expect(VerifyNoOverlap(subnets, CIDRBlock)).Should(BeNil()) - } - for _, tc := range failCases { - subnets := make([]*net.IPNet, len(tc.CIDRList)) - for i, s := range tc.CIDRList { - _, n, _ := net.ParseCIDR(s) - subnets[i] = n - } - _, CIDRBlock, _ := net.ParseCIDR(tc.CIDRBlock) - Expect(VerifyNoOverlap(subnets, CIDRBlock)).ShouldNot(BeNil(), "Test should have failed with CIDR %s\n", tc.CIDRBlock) - } - }) -}) diff --git a/pkg/util/ip.go b/pkg/util/ip.go deleted file mode 100644 index 1dd9f99d6..000000000 --- a/pkg/util/ip.go +++ /dev/null @@ -1,166 +0,0 @@ -package util - -import ( - "encoding/binary" - "fmt" - "log" - "net" - "strings" - - "github.com/kubesphere/porter/pkg/constant" - "github.com/mikioh/ipaddr" -) - -// Get preferred outbound ip of this machine -func GetOutboundIP() string { - conn, err := net.Dial("udp", "8.8.8.8:80") - if err != nil { - log.Fatal(err) - } - defer conn.Close() - localAddr := conn.LocalAddr().(*net.UDPAddr) - return localAddr.IP.String() -} - -func GetDefaultInterfaceName() string { - ip := GetOutboundIP() - if ip == "" { - return "" - } - - infs, _ := net.Interfaces() - for _, f := range infs { - addrs, err := f.Addrs() - if err != nil { - log.Fatal(err) - } - for _, addr := range addrs { - if strings.Contains(addr.String(), ip) { - return f.Name - } - } - } - return "" -} -func ToCommonString(ip string, prefix uint32) string { - return fmt.Sprintf("%s/%d", ip, prefix) -} - -func ParseAddress(addr string) ([]*net.IPNet, error) { - if strings.Contains(addr, constant.EipRangeSeparator) { - r := strings.SplitN(addr, constant.EipRangeSeparator, 2) - if len(r) != 2 { - return nil, fmt.Errorf("%s is not a valid address range", addr) - } - first := net.ParseIP(strings.TrimSpace(r[0])) - last := net.ParseIP(strings.TrimSpace(r[1])) - if first == nil || last == nil { - return nil, fmt.Errorf("%s is not a valid address range", addr) - } - - var ret []*net.IPNet - for _, pfx := range ipaddr.Summarize(first, last) { - n := &net.IPNet{ - IP: pfx.IP, - Mask: pfx.Mask, - } - ret = append(ret, n) - } - return ret, nil - } - - if !strings.Contains(addr, "/") { - addr = addr + "/32" - } - - _, ipnet, err := net.ParseCIDR(addr) - if err != nil { - return nil, err - } - - return []*net.IPNet{ipnet}, nil -} - -func GetCIDRAddressCount(cidr string) int { - ip, ipnet, err := net.ParseCIDR(cidr) - if err != nil { - if i := net.ParseIP(cidr); i != nil { - last := i.To4()[3] - if last == 255 || last == 0 { - return 0 - } - return 1 - } - return 0 - } - b, a := ipnet.Mask.Size() - size := 1 << uint(a-b) - if b <= 24 { - return size - 2< ln { - return 0 - } - - pf := (fn & 0xFFFFFF00) >> 8 - pl := (ln | 0x000000FF) >> 8 - - pad := int(lip[3]) - int(fip[3]) + 1 - - if fip[3] == 0 { - pad-- - } - - if lip[3] == 255 { - pad-- - } - - return int(int(pl-pf)*254 + pad) -} - -func GetValidAddressCount(addr string) int { - if strings.Contains(addr, constant.EipRangeSeparator) { - r := strings.SplitN(addr, constant.EipRangeSeparator, 2) - if len(r) != 2 { - return 0 - } - return GetIPRangeAddressCount(r[0], r[1]) - } - - return GetCIDRAddressCount(addr) -} - -func Intersect(n1, n2 *net.IPNet) bool { - return n2.Contains(n1.IP) || n1.Contains(n2.IP) -} diff --git a/pkg/util/ip_test.go b/pkg/util/ip_test.go deleted file mode 100644 index 48d606d42..000000000 --- a/pkg/util/ip_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package util_test - -import ( - "os" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - . "github.com/kubesphere/porter/pkg/util" -) - -var _ = Describe("Ip test", func() { - It("Should get my default gateway", func() { - ip := GetOutboundIP() - Expect(ip).ShouldNot(BeEmpty()) - nodeIP := os.Getenv("NODE_IP") - if nodeIP != "" { - Expect(ip).To(Equal(nodeIP)) - } - }) - It("Should get the default interface", func() { - name := GetDefaultInterfaceName() - Expect(name).ShouldNot(BeEmpty()) - intf := os.Getenv("DEFAULT_INTERFACE") - if intf != "" { - Expect(name).To(Equal(intf)) - } - }) - It("Should caculate correct ip address", func() { - Expect(GetValidAddressCount("192.168.1.1")).Should(Equal(1)) - Expect(GetValidAddressCount("192.168.1.0/24")).Should(Equal(254)) - Expect(GetValidAddressCount("192.168.1.1/25")).Should(Equal(127)) - Expect(GetValidAddressCount("192.168.1.0/23")).Should(Equal(508)) - Expect(GetValidAddressCount("192.168.255.255/32")).Should(Equal(0)) - Expect(GetValidAddressCount("192.168.255.255")).Should(Equal(0)) - Expect(GetValidAddressCount("192.168.255.250")).Should(Equal(1)) - Expect(GetValidAddressCount("192.168.255.0")).Should(Equal(0)) - Expect(GetValidAddressCount("192.168.0.0/16")).Should(Equal(65024)) - Expect(GetValidAddressCount("192.168.0.1-192.168.0.10")).Should(Equal(10)) - Expect(GetValidAddressCount("10.0.1.4-10.0.1.255")).Should(Equal(251)) - Expect(GetValidAddressCount("10.0.1.4-10.0.2.3")).Should(Equal(254)) - Expect(GetValidAddressCount("172.16.0.0-172.31.255.255")).Should(Equal(1<<20 - 16*256*2)) - }) -}) diff --git a/pkg/util/string.go b/pkg/util/string.go deleted file mode 100644 index 646b73e50..000000000 --- a/pkg/util/string.go +++ /dev/null @@ -1,20 +0,0 @@ -package util - -func ContainsString(slice []string, s string) bool { - for _, item := range slice { - if item == s { - return true - } - } - return false -} - -func RemoveString(slice []string, s string) (result []string) { - for _, item := range slice { - if item == s { - continue - } - result = append(result, item) - } - return -} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..b6423ab91 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,53 @@ +package util + +import ( + "github.com/kubesphere/porter/pkg/constant" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "net" + "os" +) + +func ContainsString(slice []string, s string) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +func RemoveString(slice []string, s string) (result []string) { + for _, item := range slice { + if item == s { + continue + } + result = append(result, item) + } + return +} + +// IsDeletionCandidate checks if object is candidate to be deleted +func IsDeletionCandidate(obj v1.Object, finalizer string) bool { + return obj.GetDeletionTimestamp() != nil && ContainsString(obj.GetFinalizers(), finalizer) +} + +// NeedToAddFinalizer checks if need to add finalizer to object +func NeedToAddFinalizer(obj v1.Object, finalizer string) bool { + return obj.GetDeletionTimestamp() == nil && !ContainsString(obj.GetFinalizers(), finalizer) +} + +// Find node first NodeInternalIP, should check result +func GetNodeIP(node corev1.Node) net.IP { + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + return net.ParseIP(address.Address) + } + } + + return nil +} + +func GetNodeName() string { + return os.Getenv(constant.EnvNodeName) +} diff --git a/pkg/util/util_suite_test.go b/pkg/util/util_suite_test.go index 9a34451a4..36072823b 100644 --- a/pkg/util/util_suite_test.go +++ b/pkg/util/util_suite_test.go @@ -1,6 +1,8 @@ package util_test import ( + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "testing" . "github.com/onsi/ginkgo" @@ -9,5 +11,7 @@ import ( func TestUtil(t *testing.T) { RegisterFailHandler(Fail) + log := zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)) + ctrl.SetLogger(log) RunSpecs(t, "Util Suite") } diff --git a/pkg/util/string_test.go b/pkg/util/util_test.go similarity index 100% rename from pkg/util/string_test.go rename to pkg/util/util_test.go diff --git a/pkg/validate/validate.go b/pkg/validate/validate.go index af4791661..0703953ea 100644 --- a/pkg/validate/validate.go +++ b/pkg/validate/validate.go @@ -6,8 +6,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -func IsPorterService(svc *corev1.Service) bool { - return HasPorterLBAnnotation(svc.Annotations) && IsTypeLoadBalancer(svc) +func IsPorterService(obj runtime.Object) bool { + if svc, ok := obj.(*corev1.Service); ok { + return HasPorterLBAnnotation(svc.Annotations) && IsTypeLoadBalancer(svc) + } + return false } func HasPorterLBAnnotation(annotation map[string]string) bool { @@ -23,43 +26,8 @@ func HasPorterLBAnnotation(annotation map[string]string) bool { } func IsTypeLoadBalancer(obj runtime.Object) bool { - if ser, ok := obj.(*corev1.Service); ok { - return ser.Spec.Type == corev1.ServiceTypeLoadBalancer - } - return false -} - -func IsNodeChangedWhenEndpointUpdated(a *corev1.Endpoints, b *corev1.Endpoints) bool { - if len(a.Subsets) != len(b.Subsets) { - return true - } - if len(a.Subsets) == 0 { - return false - } - if (len(a.Subsets[0].Addresses) + len(a.Subsets[0].NotReadyAddresses)) != (len(b.Subsets[0].Addresses) + len(b.Subsets[0].NotReadyAddresses)) { - return true - } - nodeMapa := make(map[string]bool) - for _, addr := range a.Subsets[0].Addresses { - nodeMapa[*addr.NodeName] = true - } - for _, addr := range a.Subsets[0].NotReadyAddresses { - nodeMapa[*addr.NodeName] = true - } - nodeMapb := make(map[string]interface{}) - for _, addr := range b.Subsets[0].Addresses { - nodeMapb[*addr.NodeName] = true - } - for _, addr := range b.Subsets[0].NotReadyAddresses { - nodeMapb[*addr.NodeName] = true - } - if len(nodeMapa) != len(nodeMapb) { - return true - } - for key := range nodeMapa { - if _, ok := nodeMapb[key]; !ok { - return true - } + if svc, ok := obj.(*corev1.Service); ok { + return svc.Spec.Type == corev1.ServiceTypeLoadBalancer } return false -} +} \ No newline at end of file