Skip to content

Commit

Permalink
Rest API server for listing and formatting devices (#649)
Browse files Browse the repository at this point in the history
  • Loading branch information
Praveenrajmani committed Sep 26, 2022
1 parent e5a1963 commit 0cda5cb
Show file tree
Hide file tree
Showing 58 changed files with 3,081 additions and 467 deletions.
4 changes: 4 additions & 0 deletions .gitignore
@@ -1,4 +1,8 @@
*~
centos*.xml
*.qcow2*
directpv
!directpv/
kubectl-directpv
!kubectl-directpv/
vdb.xml
57 changes: 57 additions & 0 deletions cmd/directpv/api-server.go
@@ -0,0 +1,57 @@
// This file is part of MinIO DirectPV
// Copyright (c) 2021, 2022 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
"context"

"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/rest"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
)

var apiServer = &cobra.Command{
Use: "api-server",
Short: "Start API server of " + consts.AppPrettyName + ".",
SilenceUsage: true,
SilenceErrors: true,
RunE: func(c *cobra.Command, args []string) error {
return startAPIServer(c.Context(), args)
},
// FIXME: Add help messages
}

func init() {
apiServer.PersistentFlags().IntVarP(&apiPort, "port", "", apiPort, "port for "+consts.AppPrettyName+" API server")
}

func startAPIServer(ctx context.Context, args []string) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()

errCh := make(chan error)
go func() {
if err := rest.ServeAPIServer(ctx, apiPort); err != nil {
klog.ErrorS(err, "unable to run API server")
errCh <- err
}
}()

return <-errCh
}
4 changes: 4 additions & 0 deletions cmd/directpv/main.go
Expand Up @@ -48,6 +48,8 @@ var (
conversionHealthzURL = ""
metricsPort = consts.MetricsPort
readinessPort = consts.ReadinessPort
apiPort = consts.APIPort
nodeAPIPort = consts.NodeAPIPort
)

var mainCmd = &cobra.Command{
Expand Down Expand Up @@ -120,6 +122,8 @@ func init() {

mainCmd.AddCommand(controllerCmd)
mainCmd.AddCommand(nodeServerCmd)
mainCmd.AddCommand(apiServer)
mainCmd.AddCommand(nodeAPIServer)
}

func main() {
Expand Down
71 changes: 71 additions & 0 deletions cmd/directpv/node-api-server.go
@@ -0,0 +1,71 @@
// This file is part of MinIO DirectPV
// Copyright (c) 2021, 2022 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
"context"
"errors"
"os"

"github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/rest"
"github.com/spf13/cobra"
"k8s.io/klog/v2"
)

var nodeAPIServer = &cobra.Command{
Use: "node-api-server",
Short: "Start Node API server of " + consts.AppPrettyName + ".",
SilenceUsage: true,
SilenceErrors: true,
RunE: func(c *cobra.Command, args []string) error {
return startNodeAPIServer(c.Context(), args)
},
// FIXME: Add help messages
}

func init() {
nodeAPIServer.PersistentFlags().IntVarP(&nodeAPIPort, "port", "", nodeAPIPort, "port for "+consts.AppPrettyName+" Node API server")
}

// ServeNodeAPIServer(ctx context.Context, nodeAPIPort int, identity, nodeID, rack, zone, region string) error {
func startNodeAPIServer(ctx context.Context, args []string) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
defer cancel()

if err := os.Mkdir(consts.MountRootDir, 0o777); err != nil && !errors.Is(err, os.ErrExist) {
return err
}

errCh := make(chan error)
go func() {
if err := rest.ServeNodeAPIServer(ctx,
nodeAPIPort,
identity,
kubeNodeName,
rack,
zone,
region,
); err != nil {
klog.ErrorS(err, "unable to run node API server")
errCh <- err
}
}()

return <-errCh
}
69 changes: 0 additions & 69 deletions cmd/directpv/node-server.go
Expand Up @@ -22,15 +22,11 @@ import (
"os"

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/google/uuid"
"github.com/minio/directpv/pkg/consts"
pkgidentity "github.com/minio/directpv/pkg/identity"
"github.com/minio/directpv/pkg/node"
"github.com/minio/directpv/pkg/sys"
"github.com/minio/directpv/pkg/volume"
"github.com/minio/directpv/pkg/xfs"
"github.com/spf13/cobra"
losetup "gopkg.in/freddierice/go-losetup.v1"
"k8s.io/klog/v2"
)

Expand All @@ -50,59 +46,6 @@ func init() {
nodeServerCmd.PersistentFlags().IntVarP(&metricsPort, "metrics-port", "", metricsPort, "Metrics port at "+consts.AppPrettyName+" exports metrics data")
}

func checkXFS(ctx context.Context, reflinkSupport bool) error {
mountPoint, err := os.MkdirTemp("", "xfs.check.mnt.")
if err != nil {
return err
}
defer os.Remove(mountPoint)

file, err := os.CreateTemp("", "xfs.check.file.")
if err != nil {
return err
}
defer os.Remove(file.Name())
file.Close()

if err = os.Truncate(file.Name(), xfs.MinSupportedDeviceSize); err != nil {
return err
}

if err = xfs.MakeFS(ctx, file.Name(), uuid.New().String(), false, reflinkSupport); err != nil {
klog.V(3).ErrorS(err, "unable to make XFS filesystem", "reflink", reflinkSupport)
return err
}

loopDevice, err := losetup.Attach(file.Name(), 0, false)
if err != nil {
return err
}

defer func() {
if err := loopDevice.Detach(); err != nil {
klog.Error(err)
}
}()

if err = xfs.Mount(loopDevice.Path(), mountPoint); err != nil {
klog.V(3).ErrorS(err, "unable to mount XFS filesystem", "reflink", reflinkSupport)
return errMountFailure
}

return sys.Unmount(mountPoint, true, true, false)
}

func getReflinkSupport(ctx context.Context) (reflinkSupport bool, err error) {
reflinkSupport = true
if err = checkXFS(ctx, reflinkSupport); err != nil {
if errors.Is(err, errMountFailure) {
reflinkSupport = false
err = checkXFS(ctx, reflinkSupport)
}
}
return
}

func startNodeServer(ctx context.Context, args []string) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
Expand All @@ -114,17 +57,6 @@ func startNodeServer(ctx context.Context, args []string) error {
}
klog.V(3).Infof("Identity server started")

reflinkSupport, err := getReflinkSupport(ctx)
if err != nil {
return err
}

if reflinkSupport {
klog.V(3).Infof("reflink support is ENABLED for XFS formatting and mounting")
} else {
klog.V(3).Infof("reflink support is DISABLED for XFS formatting and mounting")
}

errCh := make(chan error)

go func() {
Expand All @@ -142,7 +74,6 @@ func startNodeServer(ctx context.Context, args []string) error {
rack,
zone,
region,
reflinkSupport,
metricsPort,
)
if err != nil {
Expand Down
4 changes: 3 additions & 1 deletion cmd/directpv/ready.go
Expand Up @@ -38,14 +38,16 @@ func serveReadinessEndpoint(ctx context.Context) error {
return err
}

errCh := make(chan error)
go func() {
klog.V(3).Infof("Serving readiness endpoint at :%v", readinessPort)
if err := server.Serve(listener); err != nil {
klog.ErrorS(err, "unable to serve readiness endpoint")
errCh <- err
}
}()

return nil
return <-errCh
}

// readinessHandler - Checks if the process is up. Always returns success.
Expand Down
3 changes: 0 additions & 3 deletions cmd/kubectl-directpv/install.go
Expand Up @@ -39,7 +39,6 @@ var installCmd = &cobra.Command{
}

var (
admissionControl = false
image = consts.AppName + ":" + Version
registry = "quay.io"
org = "minio"
Expand All @@ -59,7 +58,6 @@ func init() {
installCmd.PersistentFlags().StringSliceVarP(&imagePullSecrets, "image-pull-secrets", "", imagePullSecrets, "Image pull secrets to be set in pod specs")
installCmd.PersistentFlags().StringVarP(&registry, "registry", "r", registry, "Registry where "+consts.AppPrettyName+" images are available")
installCmd.PersistentFlags().StringVarP(&org, "org", "g", org, "Organization name on the registry holds "+consts.AppPrettyName+" images")
installCmd.PersistentFlags().BoolVarP(&admissionControl, "admission-control", "", admissionControl, "Turn on "+consts.AppPrettyName+" admission controller")
installCmd.PersistentFlags().StringSliceVarP(&nodeSelectorParameters, "node-selector", "n", nodeSelectorParameters, "Node selector parameters")
installCmd.PersistentFlags().StringSliceVarP(&tolerationParameters, "tolerations", "t", tolerationParameters, "Tolerations parameters")
installCmd.PersistentFlags().StringVarP(&seccompProfile, "seccomp-profile", "", seccompProfile, "Set Seccomp profile")
Expand Down Expand Up @@ -95,7 +93,6 @@ func install(ctx context.Context, args []string) (err error) {
ContainerImage: image,
ContainerOrg: org,
ContainerRegistry: registry,
AdmissionControl: admissionControl,
NodeSelector: nodeSelector,
Tolerations: tolerations,
SeccompProfile: seccompProfile,
Expand Down
8 changes: 6 additions & 2 deletions go.mod
Expand Up @@ -20,7 +20,7 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.8.1
go.uber.org/multierr v1.6.0
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
golang.org/x/sys v0.0.0-20220913175220-63ea55921009
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
google.golang.org/grpc v1.40.0
gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124
Expand Down Expand Up @@ -65,6 +65,7 @@ require (
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
Expand Down Expand Up @@ -92,13 +93,16 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/yuin/goldmark v1.4.14 // indirect
go.mongodb.org/mongo-driver v1.9.1 // indirect
go.uber.org/atomic v1.7.0 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.12 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/protobuf v1.27.1 // indirect
Expand Down
18 changes: 18 additions & 0 deletions go.sum
Expand Up @@ -314,6 +314,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
Expand Down Expand Up @@ -576,6 +578,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.4.14 h1:jwww1XQfhJN7Zm+/a1ZA/3WUiEBEroYFNTiV3dKwM8U=
github.com/yuin/goldmark v1.4.14/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
Expand Down Expand Up @@ -670,6 +676,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -723,6 +731,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -822,6 +834,10 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220913175220-63ea55921009 h1:PuvuRMeLWqsf/ZdT1UUZz0syhioyv1mzuFZsXs4fvhw=
golang.org/x/sys v0.0.0-20220913175220-63ea55921009/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
Expand Down Expand Up @@ -905,6 +921,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down

0 comments on commit 0cda5cb

Please sign in to comment.