Skip to content

Commit

Permalink
Migrate DirectCSI drives/volumes at install time (#682)
Browse files Browse the repository at this point in the history
Signed-off-by: Bala.FA <bala@minio.io>

Signed-off-by: Bala.FA <bala@minio.io>
  • Loading branch information
balamurugana committed Dec 16, 2022
1 parent 42aaeda commit 5bc8801
Show file tree
Hide file tree
Showing 48 changed files with 2,118 additions and 620 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Expand Up @@ -20,7 +20,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: 1.19.4
check-latest: true
- uses: docker/setup-qemu-action@v2
- uses: docker/setup-buildx-action@v2
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/functests.yml
Expand Up @@ -20,7 +20,7 @@ jobs:
timeout-minutes: 60
strategy:
matrix:
kube-version: ['v1.18.20', 'v1.19.16', 'v1.20.15', 'v1.21.14', 'v1.22.15', 'v1.23.13', 'v1.24.7', 'v1.25.3']
kube-version: ['v1.18.20', 'v1.19.16', 'v1.20.15', 'v1.21.14', 'v1.22.16', 'v1.23.14', 'v1.24.8', 'v1.25.4']
os: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04]
exclude:
- os: ubuntu-22.04
Expand All @@ -30,7 +30,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: 1.19.4
check-latest: true

- name: Install dependencies
Expand All @@ -57,9 +57,9 @@ jobs:
docker build -t quay.io/minio/directpv:${VERSION} .
- name: Setup Minikube
uses: manusa/actions-setup-minikube@v2.7.0
uses: manusa/actions-setup-minikube@v2.7.2
with:
minikube version: 'v1.27.1'
minikube version: 'v1.28.0'
kubernetes version: ${{ matrix.kube-version }}
github token: ${{ secrets.GITHUB_TOKEN }}

Expand All @@ -73,3 +73,12 @@ jobs:
- name: Run tests
run: |
functests/run-tests.sh
- name: Run migration tests with DirectCSI v3.2.1
run: |
functests/run-migration-tests.sh "v3.2.1"
- name: Run migration tests with DirectCSI v2.0.9
if: contains(fromJson('["v1.18.20", "v1.19.16", "v1.20.15", "v1.21.14", "v1.22.16"]'), matrix.kube-version)
run: |
functests/run-migration-tests.sh "v2.0.9"
2 changes: 1 addition & 1 deletion .github/workflows/linters.yml
Expand Up @@ -20,7 +20,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: 1.19.4
check-latest: true
- uses: ludeeus/action-shellcheck@master
env:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/vulncheck.yml
Expand Up @@ -23,7 +23,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: 1.19.4
check-latest: true
- name: Install govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
Expand Down
71 changes: 71 additions & 0 deletions cmd/directpv/legacy-controller.go
@@ -0,0 +1,71 @@
// This file is part of MinIO DirectPV
// Copyright (c) 2021, 2022 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
"context"

"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/controller"
pkgidentity "github.com/minio/directpv/pkg/identity"
legacyclient "github.com/minio/directpv/pkg/legacy/client"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
)

var legacyControllerCmd = &cobra.Command{
Use: consts.LegacyControllerServerName,
Short: "Start legacy controller server.",
SilenceUsage: true,
SilenceErrors: true,
RunE: func(c *cobra.Command, args []string) error {
return startLegacyController(c.Context())
},
}

func startLegacyController(ctx context.Context) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()

idServer, err := pkgidentity.NewServer(legacyclient.Identity, Version, map[string]string{})
if err != nil {
return err
}
klog.V(3).Infof("Legacy identity server started")

ctrlServer := controller.NewLegacyServer()
klog.V(3).Infof("Legacy controller server started")

errCh := make(chan error)

go func() {
if err := runServers(ctx, csiEndpoint, idServer, ctrlServer, nil); err != nil {
klog.ErrorS(err, "unable to start GRPC servers")
errCh <- err
}
}()

go func() {
if err := serveReadinessEndpoint(ctx); err != nil {
klog.ErrorS(err, "unable to start readiness endpoint")
errCh <- err
}
}()

return <-errCh
}
71 changes: 71 additions & 0 deletions cmd/directpv/legacy-node-server.go
@@ -0,0 +1,71 @@
// This file is part of MinIO DirectPV
// Copyright (c) 2022 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
"context"

"github.com/minio/directpv/pkg/consts"
pkgidentity "github.com/minio/directpv/pkg/identity"
legacyclient "github.com/minio/directpv/pkg/legacy/client"
"github.com/minio/directpv/pkg/node"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
)

var legacyNodeServerCmd = &cobra.Command{
Use: consts.LegacyNodeServerName,
Short: "Start legacy node server.",
SilenceUsage: true,
SilenceErrors: true,
RunE: func(c *cobra.Command, args []string) error {
return startLegacyNodeServer(c.Context())
},
}

func startLegacyNodeServer(ctx context.Context) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()

idServer, err := pkgidentity.NewServer(legacyclient.Identity, Version, map[string]string{})
if err != nil {
return err
}
klog.V(3).Infof("Legacy identity server started")

errCh := make(chan error)

nodeServer := node.NewLegacyServer(nodeID, rack, zone, region)
klog.V(3).Infof("Legacy node server started")

go func() {
if err := runServers(ctx, csiEndpoint, idServer, nil, nodeServer); err != nil {
klog.ErrorS(err, "unable to start GRPC servers")
errCh <- err
}
}()

go func() {
if err := serveReadinessEndpoint(ctx); err != nil {
klog.ErrorS(err, "unable to start readiness endpoint")
errCh <- err
}
}()

return <-errCh
}
2 changes: 2 additions & 0 deletions cmd/directpv/main.go
Expand Up @@ -127,6 +127,8 @@ func init() {
mainCmd.AddCommand(nodeServerCmd)
mainCmd.AddCommand(adminServerCmd)
mainCmd.AddCommand(nodeAPIServerCmd)
mainCmd.AddCommand(legacyControllerCmd)
mainCmd.AddCommand(legacyNodeServerCmd)
}

func main() {
Expand Down
11 changes: 3 additions & 8 deletions cmd/directpv/node-server.go
Expand Up @@ -21,7 +21,6 @@ import (
"errors"
"os"

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/device"
"github.com/minio/directpv/pkg/drive"
Expand All @@ -44,15 +43,15 @@ var nodeServerCmd = &cobra.Command{
if err := device.Sync(c.Context(), nodeID); err != nil {
return err
}
return startNodeServer(c.Context(), args)
return startNodeServer(c.Context())
},
}

func init() {
nodeServerCmd.PersistentFlags().IntVar(&metricsPort, "metrics-port", metricsPort, "Metrics port at "+consts.AppPrettyName+" exports metrics data")
}

func startNodeServer(ctx context.Context, args []string) error {
func startNodeServer(ctx context.Context) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()
Expand All @@ -79,8 +78,7 @@ func startNodeServer(ctx context.Context, args []string) error {
}
}()

var nodeServer csi.NodeServer
nodeServer, err = node.NewServer(
nodeServer := node.NewServer(
ctx,
identity,
nodeID,
Expand All @@ -89,9 +87,6 @@ func startNodeServer(ctx context.Context, args []string) error {
region,
metricsPort,
)
if err != nil {
return err
}
klog.V(3).Infof("Node server started")

if err = sys.Mkdir(consts.MountRootDir, 0o755); err != nil && !errors.Is(err, os.ErrExist) {
Expand Down
47 changes: 47 additions & 0 deletions cmd/kubectl-directpv/install.go
Expand Up @@ -27,9 +27,12 @@ import (

"github.com/fatih/color"
"github.com/minio/directpv/pkg/admin"
directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/installer"
legacyclient "github.com/minio/directpv/pkg/legacy/client"
"github.com/minio/directpv/pkg/utils"
"github.com/minio/directpv/pkg/volume"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/version"
Expand All @@ -49,6 +52,7 @@ var (
disableAdminService bool
k8sVersion string
kubeVersion *version.Version
legacyFlag bool
)

var installCmd = &cobra.Command{
Expand Down Expand Up @@ -88,6 +92,7 @@ func init() {
installCmd.PersistentFlags().StringVar(&configDir, "config-dir", configDir, "Path to configuration directory")
installCmd.PersistentFlags().BoolVar(&disableAdminService, "disable-admin-service", disableAdminService, "Skip installing a node-port service for admin server")
installCmd.PersistentFlags().StringVar(&k8sVersion, "kube-version", k8sVersion, "If present, use this as kubernetes version for dry-run")
installCmd.PersistentFlags().BoolVar(&legacyFlag, "legacy", legacyFlag, "If present, include legacy services for dry-run")
addDryRunFlag(installCmd)
}

Expand Down Expand Up @@ -202,7 +207,48 @@ func getCredential() (*admin.Credential, bool, error) {
return &cred, true, nil
}

func getLegacyFlag(ctx context.Context) bool {
if dryRunFlag {
return legacyFlag
}

ctx, cancelFunc := context.WithCancel(ctx)
defer cancelFunc()

resultCh := volume.NewLister().
LabelSelector(
map[directpvtypes.LabelKey]directpvtypes.LabelValue{
directpvtypes.MigratedLabelKey: "true",
},
).
IgnoreNotFound(true).
List(ctx)
for result := range resultCh {
if result.Err != nil {
utils.Eprintf(quietFlag, true, "unable to get volumes; %v", result.Err)
break
} else {
return true
}
}

legacyclient.Init()

for result := range legacyclient.ListVolumes(ctx) {
if result.Err != nil {
utils.Eprintf(quietFlag, true, "unable to get legacy volumes; %v", result.Err)
break
} else {
return true
}
}

return false
}

func installMain(ctx context.Context) {
legacyFlag = getLegacyFlag(ctx)

auditFile := fmt.Sprintf("install.log.%v", time.Now().UTC().Format(time.RFC3339Nano))
file, err := openAuditFile(auditFile)
if err != nil {
Expand Down Expand Up @@ -241,6 +287,7 @@ func installMain(ctx context.Context) {
args.Quiet = quietFlag
args.DisableAdminService = disableAdminService
args.KubeVersion = kubeVersion
args.Legacy = legacyFlag

if err = installer.Install(ctx, args); err != nil {
utils.Eprintf(quietFlag, true, "%v\n", err)
Expand Down

0 comments on commit 5bc8801

Please sign in to comment.