Skip to content

Commit

Permalink
feat: support ApisixTls v2 (#967)
Browse files Browse the repository at this point in the history 
Co-authored-by: Jintao Zhang <tao12345666333@163.com>
  • Loading branch information
@lingsamuel @tao12345666333
lingsamuel and tao12345666333 committed May 11, 2022
1 parent 75a4166 commit e2f19b5
Show file tree
Hide file tree
Showing 16 changed files with 534 additions and 78 deletions.
8 changes: 8 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,14 @@ ifeq ("$(wildcard $(GINKGO))", "")
exit 1
endif


### push-ingress-images: Build and push Ingress image used in e2e test suites to kind or custom registry.
.PHONY: push-ingress-images
push-ingress-images:
docker build -t apache/apisix-ingress-controller:$(IMAGE_TAG) --build-arg ENABLE_PROXY=$(ENABLE_PROXY) .
docker tag apache/apisix-ingress-controller:$(IMAGE_TAG) $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)
docker push $(REGISTRY)/apache/apisix-ingress-controller:$(IMAGE_TAG)

### push-images: Push images used in e2e test suites to kind or custom registry.
.PHONY: push-images
push-images:
Expand Down
1 change: 1 addition & 0 deletions cmd/ingress/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ For example, no available LB exists in the bare metal environment.`)
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ElectionID, "election-id", config.IngressAPISIXLeader, "election id used for campaign the controller leader")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.IngressVersion, "ingress-version", config.IngressNetworkingV1, "the supported ingress api group version, can be \"networking/v1beta1\", \"networking/v1\" (for Kubernetes version v1.19.0 or higher) and \"extensions/v1beta1\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixRouteVersion, "apisix-route-version", config.ApisixRouteV2beta3, "the supported apisixroute api group version, can be \"apisix.apache.org/v2beta2\" or \"apisix.apache.org/v2beta3\"")
cmd.PersistentFlags().StringVar(&cfg.Kubernetes.ApisixTlsVersion, "apisix-tls-version", config.ApisixV2beta3, "the supported apisixtls api group version, can be \"apisix.apache.org/v2beta3\" or \"apisix.apache.org/v2\"")
cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.WatchEndpointSlices, "watch-endpointslices", false, "whether to watch endpointslices rather than endpoints")
cmd.PersistentFlags().BoolVar(&cfg.Kubernetes.EnableGatewayAPI, "enable-gateway-api", false, "whether to enable support for Gateway API")
cmd.PersistentFlags().StringVar(&cfg.APISIX.DefaultClusterBaseURL, "default-apisix-cluster-base-url", "", "the base URL of admin api / manager api for the default APISIX cluster")
Expand Down
6 changes: 6 additions & 0 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ const (
ApisixRouteV2beta3 = "apisix.apache.org/v2beta3"
// ApisixRouteV2 represents apisixroute.apisix.apache.org/v2
ApisixRouteV2 = "apisix.apache.org/v2"
// ApisixV2beta3 represents apisix.apache.org/v2beta3
ApisixV2beta3 = "apisix.apache.org/v2beta3"
// ApisixV2 represents apisix.apache.org/v2
ApisixV2 = "apisix.apache.org/v2"

_minimalResyncInterval = 30 * time.Second

Expand Down Expand Up @@ -92,6 +96,7 @@ type KubernetesConfig struct {
IngressVersion string `json:"ingress_version" yaml:"ingress_version"`
WatchEndpointSlices bool `json:"watch_endpoint_slices" yaml:"watch_endpoint_slices"`
ApisixRouteVersion string `json:"apisix_route_version" yaml:"apisix_route_version"`
ApisixTlsVersion string `json:"apisix_tls_version" yaml:"apisix_tls_version"`
EnableGatewayAPI bool `json:"enable_gateway_api" yaml:"enable_gateway_api"`
}

Expand Down Expand Up @@ -127,6 +132,7 @@ func NewDefaultConfig() *Config {
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixRouteV2beta3,
ApisixTlsVersion: ApisixV2beta3,
WatchEndpointSlices: false,
EnableGatewayAPI: false,
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func TestNewConfigFromFile(t *testing.T) {
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixRouteV2beta3,
ApisixTlsVersion: ApisixV2beta3,
},
APISIX: APISIXConfig{
DefaultClusterName: "default",
Expand Down Expand Up @@ -126,6 +127,7 @@ func TestConfigWithEnvVar(t *testing.T) {
IngressClass: IngressClass,
IngressVersion: IngressNetworkingV1,
ApisixRouteVersion: ApisixRouteV2beta3,
ApisixTlsVersion: ApisixV2beta3,
},
APISIX: APISIXConfig{
DefaultClusterName: "default",
Expand Down
185 changes: 137 additions & 48 deletions pkg/ingress/apisix_tls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package ingress

import (
"context"
"fmt"
"sync"
"time"

Expand All @@ -26,7 +27,8 @@ import (
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"

configv2beta3 "github.com/apache/apisix-ingress-controller/pkg/kube/apisix/apis/config/v2beta3"
"github.com/apache/apisix-ingress-controller/pkg/config"
"github.com/apache/apisix-ingress-controller/pkg/kube"
"github.com/apache/apisix-ingress-controller/pkg/log"
"github.com/apache/apisix-ingress-controller/pkg/types"
v1 "github.com/apache/apisix-ingress-controller/pkg/types/apisix/v1"
Expand Down Expand Up @@ -83,72 +85,133 @@ func (c *apisixTlsController) runWorker(ctx context.Context) {
}

func (c *apisixTlsController) sync(ctx context.Context, ev *types.Event) error {
key := ev.Object.(string)
event := ev.Object.(kube.ApisixTlsEvent)
key := event.Key
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
log.Errorf("found ApisixTls resource with invalid meta namespace key %s: %s", key, err)
return err
}

tls, err := c.controller.apisixTlsLister.ApisixTlses(namespace).Get(name)
var multiVersionedTls kube.ApisixTls
switch event.GroupVersion {
case config.ApisixV2beta3:
multiVersionedTls, err = c.controller.apisixTlsLister.V2beta3(namespace, name)
case config.ApisixV2:
multiVersionedTls, err = c.controller.apisixTlsLister.V2(namespace, name)
default:
return fmt.Errorf("unsupported ApisixTls group version %s", event.GroupVersion)
}

if err != nil {
if !k8serrors.IsNotFound(err) {
log.Errorf("failed to get ApisixTls %s: %s", key, err)
log.Errorw("failed to get ApisixTls",
zap.Error(err),
zap.String("key", key),
zap.String("version", event.GroupVersion),
)
return err
}
if ev.Type != types.EventDelete {
log.Warnf("ApisixTls %s was deleted before it can be delivered", key)
log.Warnw("ApisixTls %s was deleted before it can be delivered",
zap.String("key", key),
zap.String("version", event.GroupVersion),
)
// Don't need to retry.
return nil
}
}
if ev.Type == types.EventDelete {
if tls != nil {
if multiVersionedTls != nil {
// We still find the resource while we are processing the DELETE event,
// that means object with same namespace and name was created, discarding
// this stale DELETE event.
log.Warnf("discard the stale ApisixTls delete event since the %s exists", key)
return nil
}
tls = ev.Tombstone.(*configv2beta3.ApisixTls)
multiVersionedTls = ev.Tombstone.(kube.ApisixTls)
}

ssl, err := c.controller.translator.TranslateSSL(tls)
if err != nil {
log.Errorw("failed to translate ApisixTls",
zap.Error(err),
switch event.GroupVersion {
case config.ApisixV2beta3:
tls := multiVersionedTls.V2beta3()
ssl, err := c.controller.translator.TranslateSSLV2Beta3(tls)
if err != nil {
log.Errorw("failed to translate ApisixTls",
zap.Error(err),
zap.Any("ApisixTls", tls),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())
return err
}
log.Debugw("got SSL object from ApisixTls",
zap.Any("ssl", ssl),
zap.Any("ApisixTls", tls),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())
return err
}
log.Debugw("got SSL object from ApisixTls",
zap.Any("ssl", ssl),
zap.Any("ApisixTls", tls),
)

secretKey := tls.Spec.Secret.Namespace + "_" + tls.Spec.Secret.Name
c.syncSecretSSL(secretKey, key, ssl, ev.Type)
if tls.Spec.Client != nil {
caSecretKey := tls.Spec.Client.CASecret.Namespace + "_" + tls.Spec.Client.CASecret.Name
if caSecretKey != secretKey {
c.syncSecretSSL(caSecretKey, key, ssl, ev.Type)
secretKey := tls.Spec.Secret.Namespace + "_" + tls.Spec.Secret.Name
c.syncSecretSSL(secretKey, key, ssl, ev.Type)
if tls.Spec.Client != nil {
caSecretKey := tls.Spec.Client.CASecret.Namespace + "_" + tls.Spec.Client.CASecret.Name
if caSecretKey != secretKey {
c.syncSecretSSL(caSecretKey, key, ssl, ev.Type)
}
}
}

if err := c.controller.syncSSL(ctx, ssl, ev.Type); err != nil {
log.Errorw("failed to sync SSL to APISIX",
zap.Error(err),
if err := c.controller.syncSSL(ctx, ssl, ev.Type); err != nil {
log.Errorw("failed to sync SSL to APISIX",
zap.Error(err),
zap.Any("ssl", ssl),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())
return err
}
c.controller.recorderEvent(tls, corev1.EventTypeNormal, _resourceSynced, nil)
c.controller.recordStatus(tls, _resourceSynced, nil, metav1.ConditionTrue, tls.GetGeneration())
return err
case config.ApisixV2:
tls := multiVersionedTls.V2()
ssl, err := c.controller.translator.TranslateSSLV2(tls)
if err != nil {
log.Errorw("failed to translate ApisixTls",
zap.Error(err),
zap.Any("ApisixTls", tls),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())
return err
}
log.Debugw("got SSL object from ApisixTls",
zap.Any("ssl", ssl),
zap.Any("ApisixTls", tls),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())

secretKey := tls.Spec.Secret.Namespace + "_" + tls.Spec.Secret.Name
c.syncSecretSSL(secretKey, key, ssl, ev.Type)
if tls.Spec.Client != nil {
caSecretKey := tls.Spec.Client.CASecret.Namespace + "_" + tls.Spec.Client.CASecret.Name
if caSecretKey != secretKey {
c.syncSecretSSL(caSecretKey, key, ssl, ev.Type)
}
}

if err := c.controller.syncSSL(ctx, ssl, ev.Type); err != nil {
log.Errorw("failed to sync SSL to APISIX",
zap.Error(err),
zap.Any("ssl", ssl),
)
c.controller.recorderEvent(tls, corev1.EventTypeWarning, _resourceSyncAborted, err)
c.controller.recordStatus(tls, _resourceSyncAborted, err, metav1.ConditionFalse, tls.GetGeneration())
return err
}
c.controller.recorderEvent(tls, corev1.EventTypeNormal, _resourceSynced, nil)
c.controller.recordStatus(tls, _resourceSynced, nil, metav1.ConditionTrue, tls.GetGeneration())
return err
default:
return fmt.Errorf("unsupported ApisixTls group version %s", event.GroupVersion)
}
c.controller.recorderEvent(tls, corev1.EventTypeNormal, _resourceSynced, nil)
c.controller.recordStatus(tls, _resourceSynced, nil, metav1.ConditionTrue, tls.GetGeneration())
return err
}

func (c *apisixTlsController) syncSecretSSL(secretKey string, apisixTlsKey string, ssl *v1.Ssl, event types.EventType) {
Expand Down Expand Up @@ -177,10 +240,12 @@ func (c *apisixTlsController) handleSyncErr(obj interface{}, err error) {
}

event := obj.(*types.Event)
ev := event.Object.(kube.ApisixTlsEvent)
if k8serrors.IsNotFound(err) && event.Type != types.EventDelete {
log.Infow("sync ApisixTls but not found, ignore",
zap.String("event_type", event.Type.String()),
zap.String("ApisixTls", event.Object.(string)),
zap.String("ApisixTls", ev.Key),
zap.String("version", ev.GroupVersion),
)
c.workqueue.Forget(event)
return
Expand All @@ -194,6 +259,11 @@ func (c *apisixTlsController) handleSyncErr(obj interface{}, err error) {
}

func (c *apisixTlsController) onAdd(obj interface{}) {
tls, err := kube.NewApisixTls(obj)
if err != nil {
log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
return
}
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
log.Errorf("found ApisixTls object with bad namespace/name: %s, ignore it", err)
Expand All @@ -206,17 +276,28 @@ func (c *apisixTlsController) onAdd(obj interface{}) {
zap.Any("object", obj),
)
c.workqueue.Add(&types.Event{
Type: types.EventAdd,
Object: key,
Type: types.EventAdd,
Object: kube.ApisixTlsEvent{
Key: key,
GroupVersion: tls.GroupVersion(),
},
})

c.controller.MetricsCollector.IncrEvents("TLS", "add")
}

func (c *apisixTlsController) onUpdate(prev, curr interface{}) {
oldTls := prev.(*configv2beta3.ApisixTls)
newTls := curr.(*configv2beta3.ApisixTls)
if oldTls.GetResourceVersion() >= newTls.GetResourceVersion() {
oldTls, err := kube.NewApisixTls(prev)
if err != nil {
log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
return
}
newTls, err := kube.NewApisixTls(curr)
if err != nil {
log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
return
}
if oldTls.ResourceVersion() >= newTls.ResourceVersion() {
return
}
key, err := cache.MetaNamespaceKeyFunc(curr)
Expand All @@ -232,22 +313,27 @@ func (c *apisixTlsController) onUpdate(prev, curr interface{}) {
zap.Any("old object", prev),
)
c.workqueue.Add(&types.Event{
Type: types.EventUpdate,
Object: key,
Type: types.EventUpdate,
Object: kube.ApisixTlsEvent{
Key: key,
OldObject: oldTls,
GroupVersion: newTls.GroupVersion(),
},
})

c.controller.MetricsCollector.IncrEvents("TLS", "update")
}

func (c *apisixTlsController) onDelete(obj interface{}) {
tls, ok := obj.(*configv2beta3.ApisixTls)
if !ok {
tls, err := kube.NewApisixTls(obj)
if err != nil {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
return
}
tls, ok = tombstone.Obj.(*configv2beta3.ApisixTls)
if !ok {
tls, err = kube.NewApisixTls(tombstone)
if err != nil {
log.Errorw("found ApisixTls resource with bad type", zap.Error(err))
return
}
}
Expand All @@ -263,8 +349,11 @@ func (c *apisixTlsController) onDelete(obj interface{}) {
zap.Any("final state", obj),
)
c.workqueue.Add(&types.Event{
Type: types.EventDelete,
Object: key,
Type: types.EventDelete,
Object: kube.ApisixTlsEvent{
Key: key,
GroupVersion: tls.GroupVersion(),
},
Tombstone: tls,
})

Expand Down
4 changes: 2 additions & 2 deletions pkg/ingress/compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,14 @@ func (c *Controller) CompareResources(ctx context.Context) error {
// todo ApisixUpstream and ApisixPluginConfig
// ApisixUpstream and ApisixPluginConfig should be synced with ApisixRoute resource

// ApisixSSL
// ApisixSSL TODO: Support v2?
retSSL, err := c.kubeClient.APISIXClient.ApisixV2beta3().ApisixTlses(ns).List(ctx, opts)
if err != nil {
log.Error(err.Error())
ctx.Done()
} else {
for _, s := range retSSL.Items {
ssl, err := c.translator.TranslateSSL(&s)
ssl, err := c.translator.TranslateSSLV2Beta3(&s)
if err != nil {
log.Error(err.Error())
ctx.Done()
Expand Down
Loading

0 comments on commit e2f19b5

Please sign in to comment.