Skip to content

Commit

Permalink
Provisioner - fully remove obsolete code for runtime registration/der…
Browse files Browse the repository at this point in the history 
…egistration in compass (#3411)

* remove obsolete code for runtime registration deregistration in compass

* working on unit tests

* Update unit test for wait for cluster domain step

* working on unit tests for delete cluster step

* continue working on full removing compass integration

* remove director mock usage

* remove director client

* go mod fixes

* update dependencies in go.mod

* fixing unit tests for mocked uuids

* fixing unit tests and removing not used compassID

* fixing db tests - remove compassID

* fix of unit test to use mocked uuids for db testing

* removing unused deployment configuration options

* removing unused env variables from e2e tests

* remove unused method after removal of director oauth client

* remove graphql client used for director integration

* remove oauth client used for director integration

* remove unused file closer.go used for oauth client
  • Loading branch information
@koala7659
koala7659 committed May 7, 2024
1 parent 5f509e1 commit ff9c054
Show file tree
Hide file tree
Showing 42 changed files with 185 additions and 4,122 deletions.
49 changes: 1 addition & 48 deletions components/provisioner/cmd/init.go
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,17 @@
package main

import (
"crypto/tls"
"fmt"
"net/http"
"os"
"time"

"k8s.io/apimachinery/pkg/util/yaml"

"github.com/kyma-project/control-plane/components/provisioner/internal/operations/queue"

"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"

"github.com/kyma-project/control-plane/components/provisioner/internal/director"
"github.com/kyma-project/control-plane/components/provisioner/internal/gardener"
"github.com/kyma-project/control-plane/components/provisioner/internal/graphql"
"github.com/kyma-project/control-plane/components/provisioner/internal/oauth"
"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning"
"github.com/kyma-project/control-plane/components/provisioner/internal/uuid"
"github.com/pkg/errors"
ctrl "sigs.k8s.io/controller-runtime"

restclient "k8s.io/client-go/rest"
Expand All @@ -38,15 +30,13 @@ func newProvisioningService(
gardenerProject string,
provisioner provisioning.Provisioner,
dbsFactory dbsession.Factory,
directorService director.DirectorClient,
shootProvider gardener.ShootProvider,
provisioningQueue queue.OperationQueue,
deprovisioningQueue queue.OperationQueue,
shootUpgradeQueue queue.OperationQueue,
defaultEnableKubernetesVersionAutoUpdate,
defaultEnableMachineImageVersionAutoUpdate bool,
defaultEnableIMDSv2 bool,
runtimeRegistrationEnabled bool,
dynamicKubeconfigProvider DynamicKubeconfigProvider) provisioning.Service {

uuidGenerator := uuid.NewUUIDGenerator()
Expand All @@ -56,42 +46,14 @@ func newProvisioningService(
return provisioning.NewProvisioningService(
inputConverter,
graphQLConverter,
directorService,
dbsFactory,
provisioner,
uuidGenerator,
shootProvider,
provisioningQueue,
deprovisioningQueue,
shootUpgradeQueue,
dynamicKubeconfigProvider,
runtimeRegistrationEnabled)
}

func newDirectorClient(config config) (director.DirectorClient, error) {
file, err := os.ReadFile(config.DirectorOAuthPath)
if err != nil {
return nil, errors.Wrap(err, "Failed to open director config")
}

cfg := DirectorOAuth{}
err = yaml.Unmarshal(file, &cfg)
if err != nil {
return nil, errors.Wrap(err, "Failed to unmarshal director config")
}

gqlClient := graphql.NewGraphQLClient(config.DirectorURL, true, config.SkipDirectorCertVerification)
oauthClient := oauth.NewOauthClient(newHTTPClient(config.SkipDirectorCertVerification), cfg.Data.ClientID, cfg.Data.ClientSecret, cfg.Data.TokensEndpoint)

return director.NewDirectorClient(gqlClient, oauthClient), nil
}

type DirectorOAuth struct {
Data struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
TokensEndpoint string `json:"tokens_endpoint"`
} `json:"data"`
dynamicKubeconfigProvider)
}

func newShootController(gardenerNamespace string, gardenerClusterCfg *restclient.Config, dbsFactory dbsession.Factory, auditLogTenantConfigPath string) (*gardener.ShootController, error) {
Expand Down Expand Up @@ -119,12 +81,3 @@ func newGardenerClusterConfig(cfg config) (*restclient.Config, error) {

return gardenerClusterConfig, nil
}

func newHTTPClient(skipCertVerification bool) *http.Client {
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: skipCertVerification},
},
Timeout: 30 * time.Second,
}
}
71 changes: 8 additions & 63 deletions components/provisioner/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
provisioningStages "github.com/kyma-project/control-plane/components/provisioner/internal/operations/stages/provisioning"
"github.com/kyma-project/control-plane/components/provisioner/internal/persistence/database"
"github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"
"github.com/kyma-project/control-plane/components/provisioner/internal/runtime"
"github.com/kyma-project/control-plane/components/provisioner/internal/util/k8s"
"github.com/kyma-project/control-plane/components/provisioner/pkg/gqlschema"
"github.com/pkg/errors"
Expand All @@ -40,14 +39,9 @@ import (
const connStringFormat string = "host=%s port=%s user=%s password=%s dbname=%s sslmode=%s sslrootcert=%s"

type config struct {
Address string `envconfig:"default=127.0.0.1:3000"`
APIEndpoint string `envconfig:"default=/graphql"`
PlaygroundAPIEndpoint string `envconfig:"default=/graphql"`
DirectorURL string `envconfig:"default=http://compass-director.compass-system.svc.cluster.local:3000/graphql"`
SkipDirectorCertVerification bool `envconfig:"default=false"`
DirectorOAuthPath string `envconfig:"APP_DIRECTOR_OAUTH_PATH,default=./dev/director.yaml"`
RuntimeRegistrationEnabled bool `envconfig:"default=true"`
RuntimeDeregistrationEnabled bool `envconfig:"default=true"`
Address string `envconfig:"default=127.0.0.1:3000"`
APIEndpoint string `envconfig:"default=/graphql"`
PlaygroundAPIEndpoint string `envconfig:"default=/graphql"`

Database struct {
User string `envconfig:"default=postgres"`
Expand Down Expand Up @@ -78,9 +72,6 @@ type config struct {
DefaultEnableIMDSv2 bool `envconfig:"default=false"`
}

LatestDownloadedReleases int `envconfig:"default=5"`
DownloadPreReleases bool `envconfig:"default=true"`

EnqueueInProgressOperations bool `envconfig:"default=true"`

MetricsAddress string `envconfig:"default=127.0.0.1:9000"`
Expand All @@ -89,34 +80,26 @@ type config struct {
}

func (c *config) String() string {
return fmt.Sprintf("Address: %s, APIEndpoint: %s, DirectorURL: %s, "+
"SkipDirectorCertVerification: %v, DirectorOAuthPath: %s, "+
"RuntimeRegistrationEnabled %v, RuntimeDeregistrationEnabled %v, "+
return fmt.Sprintf("Address: %s, APIEndpoint: %s, "+
"DatabaseUser: %s, DatabaseHost: %s, DatabasePort: %s, "+
"DatabaseName: %s, DatabaseSSLMode: %s, "+
"ProvisioningTimeoutClusterCreation: %s "+
"ProvisioningTimeoutInstallation: %s, ProvisioningTimeoutUpgrade: %s, "+
"ProvisioningTimeoutAgentConfiguration: %s, ProvisioningTimeoutAgentConnection: %s, "+
"DeprovisioningNoInstallTimeoutClusterDeletion: %s, DeprovisioningNoInstallTimeoutWaitingForClusterDeletion: %s "+
"ShootUpgradeTimeout: %s, "+
"OperatorRoleBindingL2SubjectName: %s, OperatorRoleBindingL3SubjectName: %s, OperatorRoleBindingCreatingForAdmin: %t "+
"GardenerProject: %s, GardenerKubeconfigPath: %s, GardenerAuditLogsPolicyConfigMap: %s, AuditLogsTenantConfigPath: %s, DefaultEnableIMDSv2: %v"+
"LatestDownloadedReleases: %d, DownloadPreReleases: %v, "+
"EnqueueInProgressOperations: %v"+
"LogLevel: %s",
c.Address, c.APIEndpoint, c.DirectorURL,
c.SkipDirectorCertVerification, c.DirectorOAuthPath,
c.RuntimeDeregistrationEnabled, c.RuntimeDeregistrationEnabled,
c.Address, c.APIEndpoint,
c.Database.User, c.Database.Host, c.Database.Port,
c.Database.Name, c.Database.SSLMode,
c.ProvisioningTimeout.ClusterCreation.String(),
c.ProvisioningTimeout.Installation.String(), c.ProvisioningTimeout.Upgrade.String(),
c.ProvisioningTimeout.AgentConfiguration.String(), c.ProvisioningTimeout.AgentConnection.String(),
c.DeprovisioningTimeout.ClusterDeletion.String(), c.DeprovisioningTimeout.WaitingForClusterDeletion.String(),
c.ProvisioningTimeout.ShootUpgrade.String(),
c.OperatorRoleBinding.L2SubjectName, c.OperatorRoleBinding.L3SubjectName, c.OperatorRoleBinding.CreatingForAdmin,
c.Gardener.Project, c.Gardener.KubeconfigPath, c.Gardener.AuditLogsPolicyConfigMap, c.Gardener.AuditLogsTenantConfigPath, c.Gardener.DefaultEnableIMDSv2,
c.LatestDownloadedReleases, c.DownloadPreReleases,
c.EnqueueInProgressOperations,
c.LogLevel)
}
Expand Down Expand Up @@ -169,50 +152,14 @@ func main() {

shootClient := gardenerClientSet.Shoots(gardenerNamespace)

directorClient, err := newDirectorClient(cfg)
exitOnError(err, "Failed to initialize Director client")

k8sClientProvider := k8s.NewK8sClientProvider()

runtimeConfigurator := runtime.NewRuntimeConfigurator(k8sClientProvider, directorClient)
adminKubeconfigRequest := gardenerClient.SubResource("adminkubeconfig")
kubeconfigProvider := gardener.NewKubeconfigProvider(shootClient, adminKubeconfigRequest, secretsInterface)

var provisioningQueue queue.OperationQueue
var shootUpgradeQueue queue.OperationQueue

if cfg.RuntimeRegistrationEnabled {
provisioningQueue = queue.CreateProvisioningQueue(
cfg.ProvisioningTimeout,
dbsFactory,
directorClient,
shootClient,
cfg.OperatorRoleBinding,
k8sClientProvider,
runtimeConfigurator,
kubeconfigProvider)

shootUpgradeQueue = queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, directorClient, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)

} else {
provisioningQueue = queue.CreateProvisioningQueueWithoutRegistration(
cfg.ProvisioningTimeout,
dbsFactory,
shootClient,
cfg.OperatorRoleBinding,
k8sClientProvider,
kubeconfigProvider)

shootUpgradeQueue = queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, nil, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
}

var deprovisioningQueue queue.OperationQueue

if cfg.RuntimeDeregistrationEnabled {
deprovisioningQueue = queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, directorClient, shootClient)
} else {
deprovisioningQueue = queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, nil, shootClient)
}
provisioningQueue := queue.CreateProvisioningQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
shootUpgradeQueue := queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
deprovisioningQueue := queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, shootClient)

provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath)
shootController, err := newShootController(gardenerNamespace, gardenerClusterConfig, dbsFactory, cfg.Gardener.AuditLogsTenantConfigPath)
Expand All @@ -226,15 +173,13 @@ func main() {
cfg.Gardener.Project,
provisioner,
dbsFactory,
directorClient,
gardener.NewShootProvider(shootClient),
provisioningQueue,
deprovisioningQueue,
shootUpgradeQueue,
cfg.Gardener.DefaultEnableKubernetesVersionAutoUpdate,
cfg.Gardener.DefaultEnableMachineImageVersionAutoUpdate,
cfg.Gardener.DefaultEnableIMDSv2,
cfg.RuntimeRegistrationEnabled,
kubeconfigProvider,
)

Expand Down
15 changes: 0 additions & 15 deletions components/provisioner/e2e_test/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,6 @@ has_to_succeed

wait_for postgres 5432

export APP_DIRECTOR_URL=https://compass-gateway-auth-oauth.cmp-main.dev.kyma.cloud.sap/director/graphql
if [[ -z "$APP_DIRECTOR_OAUTH_PATH" ]]; then
export APP_DIRECTOR_OAUTH_PATH=/compass-director-secret/secret.yaml
fi

if ! [[ -f "$APP_DIRECTOR_OAUTH_PATH" ]]; then
echo "APP_DIRECTOR_OAUTH_PATH is not set or file doesn't exist $APP_DIRECTOR_OAUTH_PATH"
exit 1
fi

export APP_GARDENER_KUBECONFIG_PATH=${APP_GARDENER_KUBECONFIG_PATH:-$GARDENER_KYMA_PROW_KUBECONFIG}
export APP_GARDENER_PROJECT=${GARDENER_KYMA_PROW_PROJECT_NAME:-$APP_GARDENER_PROJECT}
Expand All @@ -102,18 +93,12 @@ export APP_DATABASE_USER=postgres

export APP_PROVISIONING_TIMEOUT_INSTALLATION=90m
export APP_PROVISIONING_TIMEOUT_UPGRADE=90m
export APP_PROVISIONING_TIMEOUT_AGENT_CONFIGURATION=90m
export APP_PROVISIONING_NO_INSTALL_TIMEOUT_AGENT_CONFIGURATION=90m
export APP_PROVISIONING_TIMEOUT_AGENT_CONNECTION=90m
export APP_PROVISIONING_TIMEOUT_CLUSTER_CREATION=90m
export APP_PROVISIONING_TIMEOUT_CLUSTER_DOMAINS=20m
export APP_PROVISIONING_NO_INSTALL_TIMEOUT_CLUSTER_CREATION=90m
export APP_PROVISIONING_TIMEOUT_UPGRADE_TRIGGERING=90m

# for testing disabled Compass registration/deregistration
export APP_RUNTIME_REGISTRATION_ENABLED="false"
export APP_RUNTIME_DEREGISTRATION_ENABLED="false"


printf '\n########## SETTING UP THE DB ##########\n\n'
go run ./pgsetup.go
Expand Down
47 changes: 10 additions & 37 deletions components/provisioner/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,37 +32,29 @@ require (

require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.6 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/containerd v1.6.8 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v20.10.24+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
Expand All @@ -73,22 +65,12 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.14 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kyma-incubator/compass/components/hydrator v0.0.0-20240311150451-fb47652e9f70 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx v1.2.29 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/sys/mount v0.3.3 // indirect
github.com/moby/sys/mountinfo v0.7.1 // indirect
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
Expand All @@ -104,27 +86,18 @@ require (
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/sosodev/duration v1.2.0 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/urfave/cli/v2 v2.27.1 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.19.0 // indirect
Expand All @@ -149,11 +122,11 @@ require (
replace (
github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.14
github.com/containerd/containerd => github.com/containerd/containerd v1.6.18
golang.org/x/crypto => golang.org/x/crypto v0.16.0
golang.org/x/net => golang.org/x/net v0.19.0
golang.org/x/sys => golang.org/x/sys v0.15.0
golang.org/x/crypto => golang.org/x/crypto v0.22.0
golang.org/x/net => golang.org/x/net v0.24.0
golang.org/x/sys => golang.org/x/sys v0.19.0
golang.org/x/text => golang.org/x/text v0.14.0
golang.org/x/tools => golang.org/x/tools v0.16.0
golang.org/x/tools => golang.org/x/tools v0.20.0
k8s.io/client-go => k8s.io/client-go v0.26.1
sourcegraph.com/sourcegraph/appdash-data => github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
)

0 comments on commit ff9c054

Please sign in to comment.