diff --git a/.gitignore b/.gitignore
index 92603ab5a..0100e2ed8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,8 @@
*~
centos*.xml
*.qcow2*
+directpv
+!directpv/
+kubectl-directpv
+!kubectl-directpv/
vdb.xml
diff --git a/cmd/directpv/api-server.go b/cmd/directpv/api-server.go
new file mode 100644
index 000000000..8ee7cf0a2
--- /dev/null
+++ b/cmd/directpv/api-server.go
@@ -0,0 +1,57 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package main
+
+import (
+ "context"
+
+ "github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/rest"
+ "github.com/spf13/cobra"
+ "k8s.io/klog/v2"
+)
+
+var apiServer = &cobra.Command{
+ Use: "api-server",
+ Short: "Start API server of " + consts.AppPrettyName + ".",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ RunE: func(c *cobra.Command, args []string) error {
+ return startAPIServer(c.Context(), args)
+ },
+ // FIXME: Add help messages
+}
+
+func init() {
+ apiServer.PersistentFlags().IntVarP(&apiPort, "port", "", apiPort, "port for "+consts.AppPrettyName+" API server")
+}
+
+func startAPIServer(ctx context.Context, args []string) error {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithCancel(ctx)
+ defer cancel()
+
+ errCh := make(chan error)
+ go func() {
+ if err := rest.ServeAPIServer(ctx, apiPort); err != nil {
+ klog.ErrorS(err, "unable to run API server")
+ errCh <- err
+ }
+ }()
+
+ return <-errCh
+}
diff --git a/cmd/directpv/main.go b/cmd/directpv/main.go
index 460cb542d..5b300f99e 100644
--- a/cmd/directpv/main.go
+++ b/cmd/directpv/main.go
@@ -48,6 +48,8 @@ var (
conversionHealthzURL = ""
metricsPort = consts.MetricsPort
readinessPort = consts.ReadinessPort
+ apiPort = consts.APIPort
+ nodeAPIPort = consts.NodeAPIPort
)
var mainCmd = &cobra.Command{
@@ -120,6 +122,8 @@ func init() {
mainCmd.AddCommand(controllerCmd)
mainCmd.AddCommand(nodeServerCmd)
+ mainCmd.AddCommand(apiServer)
+ mainCmd.AddCommand(nodeAPIServer)
}
func main() {
diff --git a/cmd/directpv/node-api-server.go b/cmd/directpv/node-api-server.go
new file mode 100644
index 000000000..eb572d648
--- /dev/null
+++ b/cmd/directpv/node-api-server.go
@@ -0,0 +1,71 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package main
+
+import (
+ "context"
+ "errors"
+ "os"
+
+ "github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/rest"
+ "github.com/spf13/cobra"
+ "k8s.io/klog/v2"
+)
+
+var nodeAPIServer = &cobra.Command{
+ Use: "node-api-server",
+ Short: "Start Node API server of " + consts.AppPrettyName + ".",
+ SilenceUsage: true,
+ SilenceErrors: true,
+ RunE: func(c *cobra.Command, args []string) error {
+ return startNodeAPIServer(c.Context(), args)
+ },
+ // FIXME: Add help messages
+}
+
+func init() {
+ nodeAPIServer.PersistentFlags().IntVarP(&nodeAPIPort, "port", "", nodeAPIPort, "port for "+consts.AppPrettyName+" Node API server")
+}
+
+// ServeNodeAPIServer(ctx context.Context, nodeAPIPort int, identity, nodeID, rack, zone, region string) error {
+func startNodeAPIServer(ctx context.Context, args []string) error {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithCancel(ctx)
+ defer cancel()
+
+ if err := os.Mkdir(consts.MountRootDir, 0o777); err != nil && !errors.Is(err, os.ErrExist) {
+ return err
+ }
+
+ errCh := make(chan error)
+ go func() {
+ if err := rest.ServeNodeAPIServer(ctx,
+ nodeAPIPort,
+ identity,
+ kubeNodeName,
+ rack,
+ zone,
+ region,
+ ); err != nil {
+ klog.ErrorS(err, "unable to run node API server")
+ errCh <- err
+ }
+ }()
+
+ return <-errCh
+}
diff --git a/cmd/directpv/node-server.go b/cmd/directpv/node-server.go
index 3faaf14d5..836a4425a 100644
--- a/cmd/directpv/node-server.go
+++ b/cmd/directpv/node-server.go
@@ -22,15 +22,11 @@ import (
"os"
"github.com/container-storage-interface/spec/lib/go/csi"
- "github.com/google/uuid"
"github.com/minio/directpv/pkg/consts"
pkgidentity "github.com/minio/directpv/pkg/identity"
"github.com/minio/directpv/pkg/node"
- "github.com/minio/directpv/pkg/sys"
"github.com/minio/directpv/pkg/volume"
- "github.com/minio/directpv/pkg/xfs"
"github.com/spf13/cobra"
- losetup "gopkg.in/freddierice/go-losetup.v1"
"k8s.io/klog/v2"
)
@@ -50,59 +46,6 @@ func init() {
nodeServerCmd.PersistentFlags().IntVarP(&metricsPort, "metrics-port", "", metricsPort, "Metrics port at "+consts.AppPrettyName+" exports metrics data")
}
-func checkXFS(ctx context.Context, reflinkSupport bool) error {
- mountPoint, err := os.MkdirTemp("", "xfs.check.mnt.")
- if err != nil {
- return err
- }
- defer os.Remove(mountPoint)
-
- file, err := os.CreateTemp("", "xfs.check.file.")
- if err != nil {
- return err
- }
- defer os.Remove(file.Name())
- file.Close()
-
- if err = os.Truncate(file.Name(), xfs.MinSupportedDeviceSize); err != nil {
- return err
- }
-
- if err = xfs.MakeFS(ctx, file.Name(), uuid.New().String(), false, reflinkSupport); err != nil {
- klog.V(3).ErrorS(err, "unable to make XFS filesystem", "reflink", reflinkSupport)
- return err
- }
-
- loopDevice, err := losetup.Attach(file.Name(), 0, false)
- if err != nil {
- return err
- }
-
- defer func() {
- if err := loopDevice.Detach(); err != nil {
- klog.Error(err)
- }
- }()
-
- if err = xfs.Mount(loopDevice.Path(), mountPoint); err != nil {
- klog.V(3).ErrorS(err, "unable to mount XFS filesystem", "reflink", reflinkSupport)
- return errMountFailure
- }
-
- return sys.Unmount(mountPoint, true, true, false)
-}
-
-func getReflinkSupport(ctx context.Context) (reflinkSupport bool, err error) {
- reflinkSupport = true
- if err = checkXFS(ctx, reflinkSupport); err != nil {
- if errors.Is(err, errMountFailure) {
- reflinkSupport = false
- err = checkXFS(ctx, reflinkSupport)
- }
- }
- return
-}
-
func startNodeServer(ctx context.Context, args []string) error {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
@@ -114,17 +57,6 @@ func startNodeServer(ctx context.Context, args []string) error {
}
klog.V(3).Infof("Identity server started")
- reflinkSupport, err := getReflinkSupport(ctx)
- if err != nil {
- return err
- }
-
- if reflinkSupport {
- klog.V(3).Infof("reflink support is ENABLED for XFS formatting and mounting")
- } else {
- klog.V(3).Infof("reflink support is DISABLED for XFS formatting and mounting")
- }
-
errCh := make(chan error)
go func() {
@@ -142,7 +74,6 @@ func startNodeServer(ctx context.Context, args []string) error {
rack,
zone,
region,
- reflinkSupport,
metricsPort,
)
if err != nil {
diff --git a/cmd/directpv/ready.go b/cmd/directpv/ready.go
index 30e882279..5446f23ee 100644
--- a/cmd/directpv/ready.go
+++ b/cmd/directpv/ready.go
@@ -38,14 +38,16 @@ func serveReadinessEndpoint(ctx context.Context) error {
return err
}
+ errCh := make(chan error)
go func() {
klog.V(3).Infof("Serving readiness endpoint at :%v", readinessPort)
if err := server.Serve(listener); err != nil {
klog.ErrorS(err, "unable to serve readiness endpoint")
+ errCh <- err
}
}()
- return nil
+ return <-errCh
}
// readinessHandler - Checks if the process is up. Always returns success.
diff --git a/cmd/kubectl-directpv/install.go b/cmd/kubectl-directpv/install.go
index 8cc705119..0c482d491 100644
--- a/cmd/kubectl-directpv/install.go
+++ b/cmd/kubectl-directpv/install.go
@@ -39,7 +39,6 @@ var installCmd = &cobra.Command{
}
var (
- admissionControl = false
image = consts.AppName + ":" + Version
registry = "quay.io"
org = "minio"
@@ -59,7 +58,6 @@ func init() {
installCmd.PersistentFlags().StringSliceVarP(&imagePullSecrets, "image-pull-secrets", "", imagePullSecrets, "Image pull secrets to be set in pod specs")
installCmd.PersistentFlags().StringVarP(®istry, "registry", "r", registry, "Registry where "+consts.AppPrettyName+" images are available")
installCmd.PersistentFlags().StringVarP(&org, "org", "g", org, "Organization name on the registry holds "+consts.AppPrettyName+" images")
- installCmd.PersistentFlags().BoolVarP(&admissionControl, "admission-control", "", admissionControl, "Turn on "+consts.AppPrettyName+" admission controller")
installCmd.PersistentFlags().StringSliceVarP(&nodeSelectorParameters, "node-selector", "n", nodeSelectorParameters, "Node selector parameters")
installCmd.PersistentFlags().StringSliceVarP(&tolerationParameters, "tolerations", "t", tolerationParameters, "Tolerations parameters")
installCmd.PersistentFlags().StringVarP(&seccompProfile, "seccomp-profile", "", seccompProfile, "Set Seccomp profile")
@@ -95,7 +93,6 @@ func install(ctx context.Context, args []string) (err error) {
ContainerImage: image,
ContainerOrg: org,
ContainerRegistry: registry,
- AdmissionControl: admissionControl,
NodeSelector: nodeSelector,
Tolerations: tolerations,
SeccompProfile: seccompProfile,
diff --git a/go.mod b/go.mod
index 683c03b32..aafdbc5c3 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/spf13/cobra v1.4.0
github.com/spf13/viper v1.8.1
go.uber.org/multierr v1.6.0
- golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8
+ golang.org/x/sys v0.0.0-20220913175220-63ea55921009
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
google.golang.org/grpc v1.40.0
gopkg.in/freddierice/go-losetup.v1 v1.0.0-20170407175016-fc9adea44124
@@ -65,6 +65,7 @@ require (
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
+ github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -92,13 +93,16 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
+ github.com/yuin/goldmark v1.4.14 // indirect
go.mongodb.org/mongo-driver v1.9.1 // indirect
go.uber.org/atomic v1.7.0 // indirect
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
- golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+ golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
+ golang.org/x/tools v0.1.12 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
google.golang.org/protobuf v1.27.1 // indirect
diff --git a/go.sum b/go.sum
index 039414e44..2a0ac0f50 100644
--- a/go.sum
+++ b/go.sum
@@ -314,6 +314,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -576,6 +578,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.4.14 h1:jwww1XQfhJN7Zm+/a1ZA/3WUiEBEroYFNTiV3dKwM8U=
+github.com/yuin/goldmark v1.4.14/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
@@ -670,6 +676,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -723,6 +731,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -822,6 +834,10 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220913175220-63ea55921009 h1:PuvuRMeLWqsf/ZdT1UUZz0syhioyv1mzuFZsXs4fvhw=
+golang.org/x/sys v0.0.0-20220913175220-63ea55921009/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
@@ -905,6 +921,8 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/pkg/apis/directpv.min.io/v1beta1/drive.go b/pkg/apis/directpv.min.io/v1beta1/drive.go
index 7009a816e..4f0f201c7 100644
--- a/pkg/apis/directpv.min.io/v1beta1/drive.go
+++ b/pkg/apis/directpv.min.io/v1beta1/drive.go
@@ -24,9 +24,9 @@ import (
// DirectPVDriveStatus denotes drive information.
type DirectPVDriveStatus struct {
Path string `json:"path"`
- TotalCapacity int64 `json:"totalCapacity"`
- AllocatedCapacity int64 `json:"allocatedCapacity"`
- FreeCapacity int64 `json:"freeCapacity"`
+ TotalCapacity int64 `json:"totalCapacity"`
+ AllocatedCapacity int64 `json:"allocatedCapacity"`
+ FreeCapacity int64 `json:"freeCapacity"`
FSUUID string `json:"fsuuid"`
NodeName string `json:"nodeName"`
Status types.DriveStatus `json:"status"`
@@ -54,11 +54,6 @@ type DirectPVDrive struct {
Status DirectPVDriveStatus `json:"status"`
}
-// // MatchGlob does glob match of nodes/drives/statuses with drive's NodeName/Path.
-// func (drive *DirectPVDrive) MatchGlob(nodes, drives []string) bool {
-// return matcher.GlobMatchNodesDrives(nodes, drives, drive.Status.NodeName, drive.Status.Path)
-// }
-
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// DirectPVDriveList denotes list of drives.
diff --git a/pkg/apis/directpv.min.io/v1beta1/volume.go b/pkg/apis/directpv.min.io/v1beta1/volume.go
index c51eafbc4..b4af2f2d0 100644
--- a/pkg/apis/directpv.min.io/v1beta1/volume.go
+++ b/pkg/apis/directpv.min.io/v1beta1/volume.go
@@ -26,9 +26,9 @@ type DirectPVVolumeStatus struct {
DriveName string `json:"driveName"`
FSUUID string `json:"fsuuid"`
NodeName string `json:"nodeName"`
- TotalCapacity int64 `json:"totalCapacity"`
- AvailableCapacity int64 `json:"availableCapacity"`
- UsedCapacity int64 `json:"usedCapacity"`
+ TotalCapacity int64 `json:"totalCapacity"`
+ AvailableCapacity int64 `json:"availableCapacity"`
+ UsedCapacity int64 `json:"usedCapacity"`
// +optional
// +patchMergeKey=type
// +patchStrategy=merge
diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go
index 702ca99cd..bdc77e931 100644
--- a/pkg/consts/consts.go
+++ b/pkg/consts/consts.go
@@ -37,6 +37,9 @@ const (
// Identity denotes identity value.
Identity = AppName + "-min-io"
+ // Namespace denotes the namespace where the app is installed
+ Namespace = Identity
+
// StorageClassName denotes storage class name.
StorageClassName = Identity
@@ -102,4 +105,29 @@ const (
// UnixCSIEndpoint is Unix CSI endpoint
UnixCSIEndpoint = "unix:///csi/csi.sock"
+
+ // APIServerContainerName is the name of the api server
+ APIServerContainerName = "api-server"
+
+ // NodeAPIServerContainerName is the name of the node api server
+ NodeAPIServerContainerName = "node-api-server"
+
+ // NodeAPIServerHLSVC denotes the name of the clusterIP service for the node API
+ NodeAPIServerHLSVC = NodeAPIServerContainerName + "-hl"
+
+ // APIPort is the default port for the api-server
+ APIPortName = "api-port"
+ APIPort = 40443
+ APIServerCertsPath = "/tmp/apiserver/certs"
+
+ // NodeAPIPort is the default port for the node-api-server
+ NodeAPIPortName = "node-api-port"
+ NodeAPIPort = 50443
+ NodeAPIServerCAPath = "/tmp/nodeapiserver/ca"
+ NodeAPIServerCertsPath = "/tmp/nodeapiserver/certs"
+
+ // key-pairs
+ PrivateKeyFileName = "key.pem"
+ PublicCertFileName = "cert.pem"
+ CACertFileName = "ca.crt"
)
diff --git a/pkg/consts/consts.go.in b/pkg/consts/consts.go.in
index 5c6d769d3..cb04d329c 100644
--- a/pkg/consts/consts.go.in
+++ b/pkg/consts/consts.go.in
@@ -35,6 +35,9 @@ const (
// Identity denotes identity value.
Identity = AppName + "-min-io"
+ // Namespace denotes the namespace where the app is installed
+ Namespace = Identity
+
// StorageClassName denotes storage class name.
StorageClassName = Identity
@@ -100,4 +103,29 @@ const (
// UnixCSIEndpoint is Unix CSI endpoint
UnixCSIEndpoint = "unix:///csi/csi.sock"
+
+ // APIServerContainerName is the name of the api server
+ APIServerContainerName = "api-server"
+
+ // NodeAPIServerContainerName is the name of the node api server
+ NodeAPIServerContainerName = "node-api-server"
+
+ // NodeAPIServerHLSVC denotes the name of the clusterIP service for the node API
+ NodeAPIServerHLSVC = NodeAPIServerContainerName + "-hl"
+
+ // APIPort is the default port for the api-server
+ APIPortName = "api-port"
+ APIPort = 40443
+ APIServerCertsPath = "/tmp/apiserver/certs"
+
+ // NodeAPIPort is the default port for the node-api-server
+ NodeAPIPortName = "node-api-port"
+ NodeAPIPort = 50443
+ NodeAPIServerCAPath = "/tmp/nodeapiserver/ca"
+ NodeAPIServerCertsPath = "/tmp/nodeapiserver/certs"
+
+ // key-pairs
+ PrivateKeyFileName = "key.pem"
+ PublicCertFileName = "cert.pem"
+ CACertFileName = "ca.crt"
)
diff --git a/pkg/device/probe.go b/pkg/device/probe.go
new file mode 100644
index 000000000..38746090f
--- /dev/null
+++ b/pkg/device/probe.go
@@ -0,0 +1,21 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+func ProbeDevices() ([]*Device, error) {
+ return probeDevices()
+}
diff --git a/pkg/device/probe_linux.go b/pkg/device/probe_linux.go
new file mode 100644
index 000000000..a3d1b84ea
--- /dev/null
+++ b/pkg/device/probe_linux.go
@@ -0,0 +1,173 @@
+//go:build linux
+
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "os"
+ "strings"
+
+ "github.com/minio/directpv/pkg/sys"
+ "k8s.io/klog/v2"
+)
+
+func probeDevices() ([]*Device, error) {
+ dir, err := os.Open(runUdevData)
+ if err != nil {
+ return nil, err
+ }
+ defer dir.Close()
+
+ names, err := dir.Readdirnames(-1)
+ if err != nil {
+ return nil, err
+ }
+
+ var devices []*Device
+ for _, name := range names {
+ if !strings.HasPrefix(name, "b") {
+ continue
+ }
+ majMinInStr := strings.TrimPrefix(name, "b")
+ major, minor, err := getMajorMinorFromStr(majMinInStr)
+ if err != nil {
+ klog.V(5).Infof("error while parsing maj:min for file: %s: %v", name, err)
+ continue
+ }
+ devName, err := getDeviceName(major, minor)
+ if err != nil {
+ klog.V(5).Infof("error while getting device name for maj:min (%v:%v): %v", major, minor, err)
+ continue
+ }
+ if isLoopBackDevice("/dev/" + devName) {
+ klog.V(5).InfoS("loopback device is ignored while syncing", "DEVNAME", devName)
+ continue
+ }
+ data, err := ReadRunUdevDataByMajorMinor(majMinInStr)
+ if err != nil {
+ klog.V(5).Infof("error while reading udevdata for device %s: %v", devName, err)
+ continue
+ }
+ device := &Device{
+ Name: devName,
+ MajorMinor: majMinInStr,
+ UDevData: data,
+ }
+ // Probe from /sys/
+ if err := device.probeSysInfo(); err != nil {
+ klog.V(5).Infof("error while probing sys info for device %s: %v", devName, err)
+ continue
+ }
+ // Probe from /proc/1/mountinfo
+ if err := device.probeMountInfo(); err != nil {
+ klog.V(5).Infof("error while probing dev info for device %s: %v", devName, err)
+ continue
+ }
+ // Probe from /proc/
+ if err := device.probeProcInfo(); err != nil {
+ klog.V(5).Infof("error while probing dev info for device %s: %v", devName, err)
+ continue
+ }
+ // Opens the device `/dev/` to probe XFS
+ if err := device.probeDevInfo(); err != nil {
+ klog.V(5).Infof("error while validating device %s: %v", devName, err)
+ continue
+ }
+ devices = append(devices, device)
+ }
+
+ return devices, nil
+}
+
+// ProbeSysInfo probes device information from /sys
+func (device *Device) probeSysInfo() (err error) {
+ device.Hidden = getHidden(device.Name)
+ if device.Removable, err = getRemovable(device.Name); err != nil {
+ return err
+ }
+
+ if device.ReadOnly, err = getReadOnly(device.Name); err != nil {
+ return err
+ }
+
+ if device.Size, err = getSize(device.Name); err != nil {
+ return err
+ }
+
+ // No partitions for hidden devices.
+ if !device.Hidden {
+ partitionNo, err := device.PartitionNumber()
+ if err != nil {
+ return err
+ }
+ if partitionNo <= 0 {
+ names, err := getPartitions(device.Name)
+ if err != nil {
+ return err
+ }
+ device.Partitioned = len(names) > 0
+ }
+ device.Holders, err = getHolders(device.Name)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// ProbeMountInfo probes mount information from /proc/1/mountinfo
+func (device *Device) probeMountInfo() (err error) {
+ _, deviceMap, err := sys.GetMounts()
+ if err != nil {
+ klog.ErrorS(err, "unable to probe mounts", "device", device.Name)
+ return err
+ }
+ device.MountPoints = deviceMap[device.Path()]
+ return nil
+}
+
+// ProbeProcInfo probes the device information from /proc
+func (device *Device) probeProcInfo() (err error) {
+ if !device.Hidden {
+ CDROMs, err := getCDROMs()
+ if err != nil {
+ return err
+ }
+ if _, found := CDROMs[device.Name]; found {
+ device.CDRom = true
+ }
+ swaps, err := getSwaps()
+ if err != nil {
+ return err
+ }
+ if _, found := swaps[device.MajorMinor]; found {
+ device.SwapOn = true
+ }
+ }
+ return nil
+}
+
+// ProbeDevInfo probes device information from /dev
+func (device *Device) probeDevInfo() (err error) {
+ // No FS information needed for hidden devices
+ if !device.Hidden && !device.CDRom {
+ return updateFSInfo(device)
+ }
+ return nil
+}
diff --git a/pkg/device/probe_other.go b/pkg/device/probe_other.go
new file mode 100644
index 000000000..b814ebd2d
--- /dev/null
+++ b/pkg/device/probe_other.go
@@ -0,0 +1,28 @@
+//go:build !linux
+
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "fmt"
+ "runtime"
+)
+
+func probeDevices() ([]*Device, error) {
+ return nil, fmt.Errorf("unsupported operating system %v", runtime.GOOS)
+}
diff --git a/pkg/sys/sys.go b/pkg/device/sys.go
similarity index 88%
rename from pkg/sys/sys.go
rename to pkg/device/sys.go
index 31682d439..fe15faf03 100644
--- a/pkg/sys/sys.go
+++ b/pkg/device/sys.go
@@ -14,8 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package sys
+package device
func GetDeviceByFSUUID(fsuuid string) (string, error) {
return getDeviceByFSUUID(fsuuid)
}
+
+func GetDeviceName(major, minor uint32) (string, error) {
+ return getDeviceName(major, minor)
+}
diff --git a/pkg/device/sys_linux.go b/pkg/device/sys_linux.go
new file mode 100644
index 000000000..be7bf8088
--- /dev/null
+++ b/pkg/device/sys_linux.go
@@ -0,0 +1,62 @@
+//go:build linux
+
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+func getDeviceByFSUUID(fsuuid string) (device string, err error) {
+ if device, err = filepath.EvalSymlinks("/dev/disk/by-uuid/" + fsuuid); err == nil {
+ device = filepath.ToSlash(device)
+ }
+ return
+}
+
+func getDeviceName(major, minor uint32) (string, error) {
+ filename := fmt.Sprintf("/sys/dev/block/%v:%v/uevent", major, minor)
+ file, err := os.Open(filename)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+
+ reader := bufio.NewReader(file)
+ for {
+ s, err := reader.ReadString('\n')
+ if err != nil {
+ return "", err
+ }
+
+ if !strings.HasPrefix(s, "DEVNAME=") {
+ continue
+ }
+
+ switch tokens := strings.SplitN(s, "=", 2); len(tokens) {
+ case 2:
+ return strings.TrimSpace(tokens[1]), nil
+ default:
+ return "", fmt.Errorf("filename %v contains invalid DEVNAME value", filename)
+ }
+ }
+}
diff --git a/pkg/sys/sys_other.go b/pkg/device/sys_other.go
similarity index 86%
rename from pkg/sys/sys_other.go
rename to pkg/device/sys_other.go
index 4c0362ce4..00844bffe 100644
--- a/pkg/sys/sys_other.go
+++ b/pkg/device/sys_other.go
@@ -16,7 +16,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package sys
+package device
import (
"fmt"
@@ -26,3 +26,7 @@ import (
func getDeviceByFSUUID(fsuuid string) (device string, err error) {
return "", fmt.Errorf("unsupported operating system %v", runtime.GOOS)
}
+
+func GetDeviceName(major, minor uint32) (string, error) {
+ return "", fmt.Errorf("unsupported operating system %v", runtime.GOOS)
+}
diff --git a/pkg/device/types.go b/pkg/device/types.go
new file mode 100644
index 000000000..dda19ce16
--- /dev/null
+++ b/pkg/device/types.go
@@ -0,0 +1,132 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "fmt"
+ "path"
+ "strconv"
+ "strings"
+
+ "github.com/minio/directpv/pkg/xfs"
+)
+
+// Device is a block device information.
+type Device struct {
+ Name string
+ MajorMinor string
+
+ // Populated from /sys
+ Hidden bool
+ Removable bool
+ ReadOnly bool
+ Size uint64
+ Partitioned bool
+ Holders []string
+
+ // Populated from /proc/1/mountinfo
+ MountPoints []string
+
+ // Populated by probing /proc/
+ SwapOn bool
+ CDRom bool
+
+ // populated by reading the device
+ FSUUID string
+ TotalCapacity uint64
+ FreeCapacity uint64
+
+ // Populated from /run/udev/data/b:
+ UDevData map[string]string
+}
+
+// DevPath return /dev notation of the path
+func (d Device) Path() string {
+ return path.Join("/dev", d.Name)
+}
+
+// FSType fetches the FSType value from the udevdata
+func (d Device) FSType() string {
+ if d.UDevData == nil {
+ return ""
+ }
+ return d.UDevData["ID_FS_TYPE"]
+}
+
+// PartitionNumber fetches the paritionNumber from the udevData
+func (d Device) PartitionNumber() (partition int, err error) {
+ if d.UDevData == nil {
+ err = fmt.Errorf("found nil udevdata for device %s", d.Name)
+ return
+ }
+ if value, found := d.UDevData["ID_PART_ENTRY_NUMBER"]; found {
+ partition, err = strconv.Atoi(value)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+func (d Device) Model() string {
+ if d.UDevData == nil {
+ return ""
+ }
+ return d.UDevData["ID_MODEL"]
+}
+
+func (d Device) Vendor() string {
+ if d.UDevData == nil {
+ return ""
+ }
+ return d.UDevData["ID_VENDOR"]
+}
+
+func (d Device) IsUnavailable() (bool, string) {
+ if d.Size < xfs.MinSupportedDeviceSize {
+ return true, fmt.Sprintf("device size less than min supported %v", xfs.MinSupportedDeviceSize)
+ }
+ if d.SwapOn {
+ return true, "device has swapOn enabled"
+ }
+ if d.Hidden {
+ return true, "hidden device"
+ }
+ if d.ReadOnly {
+ return true, "read-only device"
+ }
+ if d.Partitioned {
+ return true, "partitioned device"
+ }
+ if len(d.Holders) > 0 {
+ return true, "device has holders"
+ }
+ if len(d.MountPoints) > 0 {
+ return true, "device is mounted"
+ }
+ if isLVMMemberFSType(d.FSType()) {
+ return true, "device is a lvm member"
+ }
+ if d.CDRom {
+ return true, "device is a CDROM"
+ }
+ return false, "available device"
+}
+
+func isLVMMemberFSType(fsType string) bool {
+ return strings.EqualFold("LVM2_member", fsType)
+}
diff --git a/pkg/device/udev.go b/pkg/device/udev.go
new file mode 100644
index 000000000..6f24c1dae
--- /dev/null
+++ b/pkg/device/udev.go
@@ -0,0 +1,73 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "strings"
+)
+
+const (
+ runUdevData = "/run/udev/data"
+)
+
+// ReadRunUdevDataByMajorMinor reads udev data by major minor
+func ReadRunUdevDataByMajorMinor(majMin string) (map[string]string, error) {
+ return readRunUdevDataFile(fmt.Sprintf("%v/b%s", runUdevData, majMin))
+}
+
+func readRunUdevDataFile(filename string) (map[string]string, error) {
+ file, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ return parseRunUdevDataFile(file)
+}
+
+func parseRunUdevDataFile(r io.Reader) (map[string]string, error) {
+ reader := bufio.NewReader(r)
+ event := map[string]string{}
+ for {
+ s, err := reader.ReadString('\n')
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ return nil, err
+ }
+
+ if !strings.HasPrefix(s, "E:") {
+ continue
+ }
+
+ tokens := strings.SplitN(s, "=", 2)
+ key := strings.TrimPrefix(tokens[0], "E:")
+ switch len(tokens) {
+ case 1:
+ event[key] = ""
+ case 2:
+ event[key] = strings.TrimSpace(tokens[1])
+ }
+ }
+ return event, nil
+}
diff --git a/pkg/device/utils.go b/pkg/device/utils.go
new file mode 100644
index 000000000..7b21275a6
--- /dev/null
+++ b/pkg/device/utils.go
@@ -0,0 +1,258 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package device
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "strconv"
+ "strings"
+ "syscall"
+
+ "github.com/minio/directpv/pkg/xfs"
+ "golang.org/x/sys/unix"
+)
+
+const (
+ defaultBlockSize = 512
+)
+
+func getHidden(name string) bool {
+ // errors ignored since real devices do not have /hidden
+ // borrow idea from 'lsblk'
+ // https://github.com/util-linux/util-linux/commit/c8487d854ba5cf5bfcae78d8e5af5587e7622351
+ v, _ := readFirstLine("/sys/class/block/"+name+"/hidden", false)
+ return v == "1"
+}
+
+func getRemovable(name string) (bool, error) {
+ s, err := readFirstLine("/sys/class/block/"+name+"/removable", false)
+ return s != "" && s != "0", err
+}
+
+func getReadOnly(name string) (bool, error) {
+ s, err := readFirstLine("/sys/class/block/"+name+"/ro", false)
+ return s != "" && s != "0", err
+}
+
+func getSize(name string) (uint64, error) {
+ s, err := readFirstLine("/sys/class/block/"+name+"/size", true)
+ if err != nil {
+ return 0, err
+ }
+ ui64, err := strconv.ParseUint(s, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return ui64 * defaultBlockSize, nil
+}
+
+func getPartitions(name string) ([]string, error) {
+ names, err := readdirnames("/sys/block/"+name, false)
+ if err != nil {
+ return nil, err
+ }
+
+ partitions := []string{}
+ for _, n := range names {
+ if strings.HasPrefix(n, name) {
+ partitions = append(partitions, n)
+ }
+ }
+
+ return partitions, nil
+}
+
+func getHolders(name string) ([]string, error) {
+ return readdirnames("/sys/block/"+name+"/holders", false)
+}
+
+func getCDROMs() (map[string]struct{}, error) {
+ file, err := os.Open("/proc/sys/dev/cdrom/info")
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return map[string]struct{}{}, nil
+ }
+ return nil, err
+ }
+ defer file.Close()
+ return parseCDROMs(file)
+}
+
+func parseCDROMs(r io.Reader) (map[string]struct{}, error) {
+ reader := bufio.NewReader(r)
+ names := map[string]struct{}{}
+ for {
+ s, err := reader.ReadString('\n')
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ return nil, err
+ }
+
+ if tokens := strings.SplitAfterN(s, "drive name:", 2); len(tokens) == 2 {
+ for _, token := range strings.Fields(tokens[1]) {
+ if token != "" {
+ names[token] = struct{}{}
+ }
+ }
+ break
+ }
+ }
+ return names, nil
+}
+
+func getSwaps() (map[string]struct{}, error) {
+ file, err := os.Open("/proc/swaps")
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return map[string]struct{}{}, nil
+ }
+ return nil, err
+ }
+ defer file.Close()
+
+ reader := bufio.NewReader(file)
+
+ filenames := []string{}
+ for {
+ s, err := reader.ReadString('\n')
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ return nil, err
+ }
+
+ filenames = append(filenames, strings.Fields(s)[0])
+ }
+
+ devices := map[string]struct{}{}
+ for _, filename := range filenames[1:] {
+ major, minor, err := getDeviceMajorMinor(filename)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ return nil, err
+ }
+
+ devices[fmt.Sprintf("%v:%v", major, minor)] = struct{}{}
+ }
+ return devices, nil
+}
+
+func updateFSInfo(device *Device) error {
+ // Probe only for "xfs" devices
+ // UDev may have empty ID_FS_TYPE for xfs devices (ref: https://github.com/minio/directpv/issues/602)
+ udevFSType := device.FSType()
+ if udevFSType == "" || strings.EqualFold(udevFSType, "xfs") {
+ fsuuid, _, totalCapacity, freeCapacity, err := xfs.Probe(device.Path())
+ if err != nil && device.Size > 0 {
+ switch {
+ case errors.Is(err, xfs.ErrFSNotFound), errors.Is(err, xfs.ErrCanceled), errors.Is(err, io.ErrUnexpectedEOF):
+ default:
+ return err
+ }
+ }
+ if err != nil {
+ return err
+ }
+ device.FSUUID = fsuuid
+ device.TotalCapacity = totalCapacity
+ device.FreeCapacity = freeCapacity
+ }
+ return nil
+}
+
+func readFirstLine(filename string, errorIfNotExist bool) (string, error) {
+ getError := func(err error) error {
+ if errorIfNotExist {
+ return err
+ }
+ switch {
+ case errors.Is(err, os.ErrNotExist), errors.Is(err, os.ErrInvalid):
+ return nil
+ case strings.Contains(strings.ToLower(err.Error()), "no such device"):
+ return nil
+ case strings.Contains(strings.ToLower(err.Error()), "invalid argument"):
+ return nil
+ }
+ return err
+ }
+
+ file, err := os.Open(filename)
+ if err != nil {
+ return "", getError(err)
+ }
+ defer file.Close()
+ s, err := bufio.NewReader(file).ReadString('\n')
+ if err != nil && !errors.Is(err, io.EOF) {
+ return "", getError(err)
+ }
+ return strings.TrimSpace(s), nil
+}
+
+func readdirnames(dirname string, errorIfNotExist bool) ([]string, error) {
+ dir, err := os.Open(dirname)
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) && !errorIfNotExist {
+ err = nil
+ }
+ return nil, err
+ }
+ defer dir.Close()
+ return dir.Readdirnames(-1)
+}
+
+func getDeviceMajorMinor(device string) (major, minor uint32, err error) {
+ stat := syscall.Stat_t{}
+ if err = syscall.Stat(device, &stat); err == nil {
+ major, minor = uint32(unix.Major(stat.Rdev)), uint32(unix.Minor(stat.Rdev))
+ }
+ return
+}
+
+// getMajorMinorFromStr parses the maj:min string and extracts major and minor
+func getMajorMinorFromStr(majMin string) (major, minor uint32, err error) {
+ tokens := strings.SplitN(majMin, ":", 2)
+ if len(tokens) != 2 {
+ err = fmt.Errorf("unknown format of %v", majMin)
+ return
+ }
+
+ var major64, minor64 uint64
+ major64, err = strconv.ParseUint(tokens[0], 10, 32)
+ if err != nil {
+ return
+ }
+ major = uint32(major64)
+
+ minor64, err = strconv.ParseUint(tokens[1], 10, 32)
+ minor = uint32(minor64)
+ return
+}
+
+// isLoopBackDevice checks if the device is a loopback or not
+func isLoopBackDevice(devPath string) bool {
+ return strings.HasPrefix(path.Base(devPath), "loop")
+}
diff --git a/pkg/ellipsis/ellipsis.go b/pkg/ellipsis/ellipsis.go
new file mode 100644
index 000000000..337cfda3d
--- /dev/null
+++ b/pkg/ellipsis/ellipsis.go
@@ -0,0 +1,211 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ellipsis
+
+import (
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+var alphaRegexp = regexp.MustCompile("^[a-z]+$")
+
+func alpha2int(value string) (ui64 uint64) {
+ p := uint64(1)
+ for i := len(value) - 1; i >= 0; i-- {
+ ui64 += uint64(value[i]-96) * p
+ p *= 26
+ }
+ return ui64
+}
+
+func int2alpha(ui64 uint64) (value string) {
+ for {
+ r := ui64 % 26
+ if r == 0 {
+ r = 26
+ ui64 -= 26
+ }
+ value = string(byte(r+96)) + value
+
+ if ui64 < 26 {
+ break
+ }
+ ui64 /= 26
+ }
+
+ return value
+}
+
+type ellipsis struct {
+ start uint64
+ end uint64
+ isAlpha bool
+
+ startIndex int
+ endIndex int
+
+ current uint64
+ prefix string
+ suffix string
+ next *ellipsis
+}
+
+func (e *ellipsis) reset() {
+ e.current = e.start
+}
+
+func (e *ellipsis) get(prefix string) string {
+ if e.current > e.end {
+ return ""
+ }
+
+ value := fmt.Sprintf("%v", e.current)
+ if e.isAlpha {
+ value = int2alpha(e.current)
+ }
+ value = prefix + e.prefix + value + e.suffix
+
+ if e.next != nil {
+ if newValue := e.next.get(value); newValue != "" {
+ return newValue
+ }
+
+ e.next.reset()
+ e.current++
+ return e.get(prefix)
+ }
+
+ e.current++
+ return value
+}
+
+func (e *ellipsis) expand() (result []string) {
+ var value string
+ for {
+ if value = e.get(""); value == "" {
+ break
+ }
+ result = append(result, value)
+ }
+ return result
+}
+
+func parseEllipsis(arg string, start, end int) (*ellipsis, error) {
+ pattern := arg[start:end]
+ parseValue := func(value string) (ui64 uint64, isAlpha bool, err error) {
+ if ui64, err = strconv.ParseUint(value, 10, 64); err == nil {
+ return ui64, false, nil
+ }
+
+ if alphaRegexp.MatchString(value) {
+ return alpha2int(value), true, nil
+ }
+ return 0, false, err
+ }
+
+ tokens := strings.Split(arg[start+1:end-1], "...")
+ switch len(tokens) {
+ case 0, 1:
+ return nil, fmt.Errorf("%v: invalid ellipsis %v at %v", arg, pattern, start)
+ }
+
+ startValue, isAlphaStart, err := parseValue(tokens[0])
+ if err != nil {
+ return nil, fmt.Errorf("%v: invalid start value '%v' in ellipsis %v at %v", arg, tokens[0], pattern, start)
+ }
+
+ endValue, isAlphaEnd, err := parseValue(tokens[1])
+ if err != nil {
+ return nil, fmt.Errorf("%v: invalid end value '%v' in ellipsis %v at %v", arg, tokens[1], pattern, start)
+ }
+
+ if isAlphaStart != isAlphaEnd {
+ return nil, fmt.Errorf("%v: invalid ellipsis %v at %v; start/end must be same kind", arg, pattern, start)
+ }
+
+ if startValue > endValue {
+ startValue, endValue = endValue, startValue
+ }
+
+ return &ellipsis{
+ start: startValue,
+ end: endValue,
+ isAlpha: isAlphaStart,
+ startIndex: start,
+ endIndex: end,
+ current: startValue,
+ }, nil
+}
+
+func getEllipses(arg string) (ellipses []*ellipsis, err error) {
+ curlyOpened := false
+ start := 0
+ for i, c := range arg {
+ switch c {
+ case '{':
+ if curlyOpened {
+ return nil, fmt.Errorf("%v: nested ellipsis pattern at %v", arg, i+1)
+ }
+
+ curlyOpened = true
+ start = i
+
+ case '}':
+ if !curlyOpened {
+ return nil, fmt.Errorf("%v: invalid ellipsis pattern at %v", arg, i+1)
+ }
+ curlyOpened = false
+
+ ellipsis, err := parseEllipsis(arg, start, i+1)
+ if err != nil {
+ return nil, err
+ }
+
+ ellipses = append(ellipses, ellipsis)
+ }
+ }
+
+ return ellipses, nil
+}
+
+// Expand expends ellipses of given argument.
+func Expand(arg string) ([]string, error) {
+ ellipses, err := getEllipses(arg)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(ellipses) == 0 {
+ return []string{arg}, nil
+ }
+
+ startIndex := 0
+ var prev *ellipsis
+ for _, e := range ellipses {
+ e.prefix = arg[startIndex:e.startIndex]
+ if prev != nil {
+ prev.next = e
+ }
+ startIndex = e.endIndex
+ prev = e
+ }
+ ellipses[len(ellipses)-1].suffix = arg[startIndex:]
+
+ return ellipses[0].expand(), nil
+}
diff --git a/pkg/ellipsis/ellipsis_test.go b/pkg/ellipsis/ellipsis_test.go
new file mode 100644
index 000000000..b77e72c55
--- /dev/null
+++ b/pkg/ellipsis/ellipsis_test.go
@@ -0,0 +1,159 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package ellipsis
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestExpand(t *testing.T) {
+ testCases := []struct {
+ input string
+ output []string
+ errReturned bool
+ }{
+ // Valid case - Start with ellipsis
+ {"{a...c}", []string{"a", "b", "c"}, false},
+ // Valid case - Start with ellipsis
+ {"{f...c}", []string{"c", "d", "e", "f"}, false},
+ // Valid case - Start with ellipsis
+ {"{az...bc}", []string{"az", "ba", "bb", "bc"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}a", []string{"aa", "ba", "ca"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}a1", []string{"aa1", "ba1", "ca1"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}{0...2}", []string{"a0", "a1", "a2", "b0", "b1", "b2", "c0", "c1", "c2"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}p{0...2}", []string{"ap0", "ap1", "ap2", "bp0", "bp1", "bp2", "cp0", "cp1", "cp2"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}p{0...2}9", []string{"ap09", "ap19", "ap29", "bp09", "bp19", "bp29", "cp09", "cp19", "cp29"}, false},
+ // Valid case- Start with ellipsis
+ {"{a...c}p{0...2}9{d...a}", []string{
+ "ap09a", "ap09b", "ap09c", "ap09d", "ap19a", "ap19b", "ap19c", "ap19d", "ap29a",
+ "ap29b", "ap29c", "ap29d", "bp09a", "bp09b", "bp09c", "bp09d", "bp19a", "bp19b", "bp19c", "bp19d", "bp29a",
+ "bp29b", "bp29c", "bp29d", "cp09a", "cp09b", "cp09c", "cp09d", "cp19a", "cp19b", "cp19c", "cp19d", "cp29a", "cp29b", "cp29c", "cp29d",
+ }, false},
+ // Valid case- Start with non-ellipsis
+ {"abc", []string{"abc"}, false},
+ // Valid case- Start with non-ellipsis
+ {"ab{p...r}", []string{"abp", "abq", "abr"}, false},
+ // Valid case- Start with non-ellipsis
+ {"ab{p...r}1", []string{"abp1", "abq1", "abr1"}, false},
+ // Valid case- Start with non-ellipsis
+ {"ab{p...r}0{1...2}", []string{"abp01", "abp02", "abq01", "abq02", "abr01", "abr02"}, false},
+ // Valid case- ellipsis start with two digit
+ {"a{12...20}x", []string{"a12x", "a13x", "a14x", "a15x", "a16x", "a17x", "a18x", "a19x", "a20x"}, false},
+ // Valid case - ellipsis start with two digit end with two digits
+ {"ax{ab...dx}y", []string{
+ "axaby", "axacy", "axady", "axaey", "axafy", "axagy", "axahy", "axaiy", "axajy", "axaky",
+ "axaly", "axamy", "axany", "axaoy", "axapy", "axaqy", "axary", "axasy", "axaty", "axauy", "axavy", "axawy", "axaxy", "axayy", "axazy",
+ "axbay", "axbby", "axbcy", "axbdy", "axbey", "axbfy", "axbgy", "axbhy", "axbiy", "axbjy", "axbky", "axbly", "axbmy", "axbny", "axboy",
+ "axbpy", "axbqy", "axbry", "axbsy", "axbty", "axbuy", "axbvy", "axbwy", "axbxy", "axbyy", "axbzy", "axcay", "axcby", "axccy", "axcdy",
+ "axcey", "axcfy", "axcgy", "axchy", "axciy", "axcjy", "axcky", "axcly", "axcmy", "axcny", "axcoy", "axcpy", "axcqy", "axcry", "axcsy",
+ "axcty", "axcuy", "axcvy", "axcwy", "axcxy", "axcyy", "axczy", "axday", "axdby", "axdcy", "axddy", "axdey", "axdfy", "axdgy", "axdhy",
+ "axdiy", "axdjy", "axdky", "axdly", "axdmy", "axdny", "axdoy", "axdpy", "axdqy", "axdry", "axdsy", "axdty", "axduy", "axdvy", "axdwy", "axdxy",
+ }, false},
+ // Invalid case with one dot
+ {"a{a.c}p", nil, true},
+ // Invalid case - two dots
+ {"a{a..c}p", nil, true},
+ // Invalid case - four dots
+ {"a{a....c}p", nil, true},
+ }
+ for i, test := range testCases {
+ expansion, err := Expand(test.input)
+ errReturned := err != nil
+ if errReturned != test.errReturned {
+ t.Fatalf("Test %d: expected %t got %t", i+1, test.errReturned, errReturned)
+ }
+ if !reflect.DeepEqual(expansion, test.output) {
+ t.Fatalf("Test %d: expected %s got %s", i+1, test.output, expansion)
+ }
+ }
+}
+
+func TestGetEllipsis(t *testing.T) {
+ testCases := []struct {
+ arg string
+ ellipses []*ellipsis
+ errReturned bool
+ }{
+ // Valid case
+ {"{a...z}", []*ellipsis{{start: 1, end: 26, isAlpha: true, startIndex: 0, endIndex: 7}}, false},
+ // Valid case
+ {"{aa...az}", []*ellipsis{{start: 27, end: 52, isAlpha: true, startIndex: 0, endIndex: 9}}, false},
+ // Valid case
+ {"{0...11}", []*ellipsis{{start: 0, end: 11, isAlpha: false, startIndex: 0, endIndex: 8}}, false},
+ // Alpha numeric combination
+ {"{a0...z}", []*ellipsis{}, true},
+ // One dot in expansion
+ {"{a.z}", []*ellipsis{}, true},
+ // Two dot in expansion
+ {"{a..z}", []*ellipsis{}, true},
+ // Four or more dots in expansion
+ {"{a....z}", []*ellipsis{}, true},
+ // No dot in expansion
+ {"{123}", []*ellipsis{}, true},
+ // Multiple opening braces in ellipsis
+ {"{a...{a...z}}", []*ellipsis{}, true},
+ // No RHS
+ {"{a...}z", []*ellipsis{}, true},
+ // No LHS
+ {"{...b}z", []*ellipsis{}, true},
+ // Multiple openin braces
+ {"{1.{...{zz}", []*ellipsis{}, true},
+ // Invalid numer of braces
+ {"1}ccc{sss}", []*ellipsis{}, true},
+ // Alphabet in LHS number in RHS
+ {"{11...az}", []*ellipsis{}, true},
+ // Alphabet in LHS number in RHS
+ {"{a...0}", []*ellipsis{}, true},
+ // Number in LHS alphabet in RHS
+ {"{0...a}", []*ellipsis{}, true},
+ // alphabet in LHS and Number in RHS
+ {"{a...0}", []*ellipsis{}, true},
+ }
+
+ for i, test := range testCases {
+ ellipses, err := getEllipses(test.arg)
+ errReturned := err != nil
+ if errReturned != test.errReturned {
+ t.Fatalf("Test %d: expected %t got %t", i+1, test.errReturned, errReturned)
+ }
+
+ for index, p := range ellipses {
+ ts := test.ellipses[index]
+ if p.start != ts.start {
+ t.Fatalf("Test %d: expected %d got %d", i+1, ts.start, p.start)
+ }
+ if p.end != ts.end {
+ t.Fatalf("Test %d: expected %d got %d", i+1, ts.end, p.end)
+ }
+ if p.isAlpha != ts.isAlpha {
+ t.Fatalf("Test %d: expected %t got %t", i+1, ts.isAlpha, p.isAlpha)
+ }
+ if p.startIndex != ts.startIndex {
+ t.Fatalf("Test %d: expected %d got %d", i+1, ts.startIndex, p.startIndex)
+ }
+ if p.endIndex != ts.endIndex {
+ t.Fatalf("Test %d: expected %d got %d", i+1, ts.endIndex, p.endIndex)
+ }
+ }
+ }
+}
diff --git a/pkg/installer/api-server-deployment.go b/pkg/installer/api-server-deployment.go
new file mode 100644
index 000000000..465482017
--- /dev/null
+++ b/pkg/installer/api-server-deployment.go
@@ -0,0 +1,170 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package installer
+
+import (
+ "context"
+ "fmt"
+ "path"
+
+ "github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/k8s"
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func installAPIServerDeploymentDefault(ctx context.Context, c *Config) error {
+ deploymentsClient := k8s.KubeClient().AppsV1().Deployments(c.namespace())
+ if _, err := deploymentsClient.Get(ctx, c.apiServerDeploymentName(), metav1.GetOptions{}); err != nil {
+ if !apierrors.IsNotFound(err) {
+ return err
+ }
+ } else {
+ // Deployment already created
+ return nil
+ }
+ return createAPIServerDeployment(ctx, c)
+}
+
+func uninstallAPIServerDeploymentDefault(ctx context.Context, c *Config) error {
+ if err := deleteDeployment(ctx, c.namespace(), c.apiServerDeploymentName()); err != nil && !apierrors.IsNotFound(err) {
+ return err
+ }
+ if err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Delete(ctx, apiServerCertsSecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ return err
+ }
+ if err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Delete(ctx, apiServerCASecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ return err
+ }
+ return nil
+}
+
+func createAPIServerDeployment(ctx context.Context, c *Config) error {
+ // Create cert secrets for the api-server
+ if err := generateCertSecretsForAPIServer(ctx, c); err != nil {
+ return err
+ }
+ // Create api-server deployment
+ var replicas int32 = 1
+ privileged := false
+ podSpec := corev1.PodSpec{
+ ServiceAccountName: c.serviceAccountName(),
+ Volumes: []corev1.Volume{
+ newSecretVolume(apiServerCertsDir, apiServerCertsSecretName),
+ newSecretVolume(nodeAPIServerCADir, nodeAPIServerCASecretName),
+ },
+ ImagePullSecrets: c.getImagePullSecrets(),
+ Containers: []corev1.Container{
+ {
+ Name: consts.APIServerContainerName,
+ Image: path.Join(c.ContainerRegistry, c.ContainerOrg, c.ContainerImage),
+ Args: []string{
+ "api-server",
+ fmt.Sprintf("-v=%d", logLevel),
+ fmt.Sprintf("--identity=%s", c.identity()),
+ fmt.Sprintf("--port=%d", consts.APIPort),
+ fmt.Sprintf("--csi-endpoint=$(%s)", csiEndpointEnvVarName),
+ fmt.Sprintf("--kube-node-name=$(%s)", kubeNodeNameEnvVarName),
+ },
+ SecurityContext: &corev1.SecurityContext{
+ Privileged: &privileged,
+ },
+ // ReadinessProbe: &corev1.Probe{ProbeHandler: getReadinessHandler()},
+ Env: []corev1.EnvVar{kubeNodeNameEnvVar},
+ VolumeMounts: []corev1.VolumeMount{
+ newVolumeMount(apiServerCertsDir, consts.APIServerCertsPath, corev1.MountPropagationNone, false),
+ newVolumeMount(nodeAPIServerCADir, consts.NodeAPIServerCAPath, corev1.MountPropagationNone, false),
+ },
+ Ports: []corev1.ContainerPort{
+ {
+ ContainerPort: consts.APIPort,
+ Name: consts.APIPortName,
+ Protocol: corev1.ProtocolTCP,
+ },
+ },
+ },
+ },
+ }
+
+ generatedSelectorValue := generateSanitizedUniqueNameFrom(c.apiServerDeploymentName())
+ deployment := &appsv1.Deployment{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "apps/v1",
+ Kind: "Deployment",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: c.apiServerDeploymentName(),
+ Namespace: c.namespace(),
+ Annotations: defaultAnnotations,
+ Labels: defaultLabels,
+ },
+ Spec: appsv1.DeploymentSpec{
+ Replicas: &replicas,
+ Selector: metav1.AddLabelToSelector(&metav1.LabelSelector{}, selectorKey, generatedSelectorValue),
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: c.apiServerDeploymentName(),
+ Namespace: c.namespace(),
+ Annotations: map[string]string{
+ createdByLabel: pluginName,
+ },
+ Labels: map[string]string{
+ selectorKey: generatedSelectorValue,
+ },
+ },
+ Spec: podSpec,
+ },
+ },
+ Status: appsv1.DeploymentStatus{},
+ }
+ deployment.Finalizers = []string{
+ c.namespace() + deleteProtectionFinalizer,
+ }
+
+ if !c.DryRun {
+ if _, err := k8s.KubeClient().AppsV1().Deployments(c.namespace()).Create(ctx, deployment, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ }
+
+ return c.postProc(deployment)
+}
+
+func generateCertSecretsForAPIServer(ctx context.Context, c *Config) error {
+ caCertBytes, publicCertBytes, privateKeyBytes, certErr := getCerts([]string{
+ localHostDNS,
+ // FIXME: Add nodeport svc domain name here
+ })
+ if certErr != nil {
+ return certErr
+ }
+ return createOrUpdateAPIServerSecrets(ctx, caCertBytes, publicCertBytes, privateKeyBytes, c)
+}
+
+func createOrUpdateAPIServerSecrets(ctx context.Context, caCertBytes, publicCertBytes, privateKeyBytes []byte, c *Config) error {
+ if err := createOrUpdateSecret(ctx, apiServerCertsSecretName, map[string][]byte{
+ consts.PrivateKeyFileName: privateKeyBytes,
+ consts.PublicCertFileName: publicCertBytes,
+ }, c); err != nil {
+ return err
+ }
+ return createOrUpdateSecret(ctx, apiServerCASecretName, map[string][]byte{
+ consts.CACertFileName: caCertBytes,
+ }, c)
+}
diff --git a/pkg/installer/certs.go b/pkg/installer/certs.go
index a0e634bb1..37a927d69 100644
--- a/pkg/installer/certs.go
+++ b/pkg/installer/certs.go
@@ -32,12 +32,7 @@ func getCerts(dnsNames []string) (caCertBytes, publicCertBytes, privateKeyBytes
ca := &x509.Certificate{
SerialNumber: big.NewInt(2019),
Subject: pkix.Name{
- Organization: []string{"MinIO, Inc."},
- Country: []string{"US"},
- Province: []string{"CA"},
- Locality: []string{"Redwood City"},
- StreetAddress: []string{"275 Shoreline Dr, Ste 100,"},
- PostalCode: []string{"94065"},
+ Organization: []string{"MinIO, Inc."},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
@@ -81,12 +76,7 @@ func getCerts(dnsNames []string) (caCertBytes, publicCertBytes, privateKeyBytes
cert := &x509.Certificate{
SerialNumber: big.NewInt(2019),
Subject: pkix.Name{
- Organization: []string{"MinIO, Inc."},
- Country: []string{"US"},
- Province: []string{"CA"},
- Locality: []string{"Redwood City"},
- StreetAddress: []string{"275 Shoreline Dr, Ste 100,"},
- PostalCode: []string{"94065"},
+ Organization: []string{"MinIO, Inc."},
},
DNSNames: dnsNames,
NotBefore: time.Now(),
diff --git a/pkg/installer/config.go b/pkg/installer/config.go
index 2b0ddefc4..2a7e289c1 100644
--- a/pkg/installer/config.go
+++ b/pkg/installer/config.go
@@ -60,9 +60,6 @@ type Config struct {
NodeDriverRegistrarImage string
LivenessProbeImage string
- // Admission controller
- AdmissionControl bool
-
// Selectors and tolerations
NodeSelector map[string]string
Tolerations []corev1.Toleration
@@ -83,9 +80,6 @@ type Config struct {
// Image pull secrets
ImagePullSecrets []string
-
- // internal
- validationWebhookCaBundle []byte
}
type installer interface {
@@ -105,10 +99,6 @@ func (c *Config) namespace() string {
return c.Identity
}
-func (c *Config) serviceName() string {
- return c.Identity
-}
-
func (c *Config) identity() string {
return c.Identity
}
@@ -137,6 +127,10 @@ func (c *Config) deploymentName() string {
return "controller"
}
+func (c *Config) apiServerDeploymentName() string {
+ return "api-server"
+}
+
func (c *Config) getPSPName() string {
return c.Identity
}
diff --git a/pkg/installer/const.go b/pkg/installer/const.go
index 4039b237b..3335f1c06 100644
--- a/pkg/installer/const.go
+++ b/pkg/installer/const.go
@@ -47,17 +47,7 @@ const (
volumePathRunUdevData = consts.UdevDataDir
// Deployment
- admissionWebhookSecretName = "validationwebhookcerts"
- admissionControllerWebhookPort = 20443
- admissionControllerWebhookName = "validatinghook"
- validationControllerName = consts.AppName + "-validation-controller"
- admissionControllerCertsDir = "admission-webhook-certs"
- admissionCertsDir = "/etc/admission/certs"
- csiProvisionerContainerName = "csi-provisioner"
- admissionWehookDNSName = consts.AppName + "-validation-controller." + consts.Identity + ".svc"
-
- // validation rules
- validationWebhookConfigName = "drive.validation.controller"
+ csiProvisionerContainerName = "csi-provisioner"
// Common
volumeNameSocketDir = "socket-dir"
@@ -77,10 +67,6 @@ const (
// debug log level default
logLevel = 3
- // key-pairs
- privateKeyFileName = "key.pem"
- publicCertFileName = "cert.pem"
-
// string-gen
charset = "abcdefghijklmnopqrstuvwxyz0123456789"
@@ -94,4 +80,16 @@ const (
// readiness
readinessPortName = "readinessport"
+
+ // api-server
+ apiServerCertsDir = "api-server-certs"
+ apiServerCertsSecretName = "apiservercerts"
+ localHostDNS = "localhost"
+ apiServerCASecretName = "apiservercacert"
+
+ // node-api-server
+ nodeAPIServerCertsDir = "node-api-server-certs"
+ nodeAPIServerCertsSecretName = "nodeapiservercerts"
+ nodeAPIServerCASecretName = "nodeapiservercacert"
+ nodeAPIServerCADir = "node-api-server-ca"
)
diff --git a/pkg/installer/daemonset.go b/pkg/installer/daemonset.go
index 5af166b89..ff98e98c7 100644
--- a/pkg/installer/daemonset.go
+++ b/pkg/installer/daemonset.go
@@ -31,20 +31,37 @@ import (
)
func installDaemonsetDefault(ctx context.Context, c *Config) error {
- if err := createDaemonSet(ctx, c); err != nil {
- return err
+ daemonSetsClient := k8s.KubeClient().AppsV1().DaemonSets(c.namespace())
+ if _, err := daemonSetsClient.Get(ctx, c.daemonsetName(), metav1.GetOptions{}); err != nil {
+ if !apierrors.IsNotFound(err) {
+ return err
+ }
+ } else {
+ // Deployment already created
+ return nil
}
- return nil
+ return createDaemonSet(ctx, c)
}
func uninstallDaemonsetDefault(ctx context.Context, c *Config) error {
if err := k8s.KubeClient().AppsV1().DaemonSets(c.namespace()).Delete(ctx, c.daemonsetName(), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
return err
}
+ if err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Delete(ctx, nodeAPIServerCertsSecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ return err
+ }
+ if err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Delete(ctx, nodeAPIServerCASecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ return err
+ }
return nil
}
func createDaemonSet(ctx context.Context, c *Config) error {
+ // Create cert secrets for the node api-server
+ if err := generateCertSecretsForNodeAPIServer(ctx, c); err != nil {
+ return err
+ }
+ // Create deamonset
privileged := true
securityContext := &corev1.SecurityContext{Privileged: &privileged}
@@ -65,6 +82,8 @@ func createDaemonSet(ctx context.Context, c *Config) error {
newHostPathVolume(volumeNameSysDir, volumePathSysDir),
newHostPathVolume(volumeNameDevDir, volumePathDevDir),
newHostPathVolume(volumeNameRunUdevData, volumePathRunUdevData),
+ // node api server
+ newSecretVolume(nodeAPIServerCertsDir, nodeAPIServerCertsSecretName),
}
volumeMounts := []corev1.VolumeMount{
newVolumeMount(volumeNameSocketDir, socketDir, corev1.MountPropagationNone, false),
@@ -139,6 +158,44 @@ func createDaemonSet(ctx context.Context, c *Config) error {
},
},
},
+ {
+ Name: consts.NodeAPIServerContainerName,
+ Image: path.Join(c.ContainerRegistry, c.ContainerOrg, c.ContainerImage),
+ Args: func() []string {
+ args := []string{
+ "node-api-server",
+ fmt.Sprintf("-v=%d", logLevel),
+ fmt.Sprintf("--kube-node-name=$(%s)", kubeNodeNameEnvVarName),
+ fmt.Sprintf("--port=%d", consts.NodeAPIPort),
+ }
+ return args
+ }(),
+ SecurityContext: securityContext,
+ Env: []corev1.EnvVar{kubeNodeNameEnvVar},
+ TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
+ TerminationMessagePath: "/var/log/driver-termination-log",
+ VolumeMounts: append(volumeMounts, newVolumeMount(nodeAPIServerCertsDir, consts.NodeAPIServerCertsPath, corev1.MountPropagationNone, false)),
+ Ports: []corev1.ContainerPort{
+ {
+ ContainerPort: consts.NodeAPIPort,
+ Name: consts.NodeAPIPortName,
+ Protocol: corev1.ProtocolTCP,
+ },
+ },
+ /*ReadinessProbe: &corev1.Probe{ProbeHandler: getReadinessHandler()},
+ LivenessProbe: &corev1.Probe{
+ FailureThreshold: 5,
+ InitialDelaySeconds: 300,
+ TimeoutSeconds: 5,
+ PeriodSeconds: 5,
+ ProbeHandler: corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{
+ Path: healthZContainerPortPath,
+ Port: intstr.FromString(healthZContainerPortName),
+ },
+ },
+ },*/
+ },
{
Name: livenessProbeContainerName,
Image: path.Join(c.ContainerRegistry, c.ContainerOrg, c.getLivenessProbeImage()),
@@ -205,3 +262,26 @@ func createDaemonSet(ctx context.Context, c *Config) error {
}
return c.postProc(daemonset)
}
+
+func generateCertSecretsForNodeAPIServer(ctx context.Context, c *Config) error {
+ caCertBytes, publicCertBytes, privateKeyBytes, certErr := getCerts([]string{
+ localHostDNS,
+ // FIXME: Add clusterIP svc domain name here
+ })
+ if certErr != nil {
+ return certErr
+ }
+ return createOrUpdateNodeAPIServerSecrets(ctx, caCertBytes, publicCertBytes, privateKeyBytes, c)
+}
+
+func createOrUpdateNodeAPIServerSecrets(ctx context.Context, caCertBytes, publicCertBytes, privateKeyBytes []byte, c *Config) error {
+ if err := createOrUpdateSecret(ctx, nodeAPIServerCertsSecretName, map[string][]byte{
+ consts.PrivateKeyFileName: privateKeyBytes,
+ consts.PublicCertFileName: publicCertBytes,
+ }, c); err != nil {
+ return err
+ }
+ return createOrUpdateSecret(ctx, nodeAPIServerCASecretName, map[string][]byte{
+ consts.CACertFileName: caCertBytes,
+ }, c)
+}
diff --git a/pkg/installer/default.go b/pkg/installer/default.go
index 5c91860f6..87d1b47e4 100644
--- a/pkg/installer/default.go
+++ b/pkg/installer/default.go
@@ -164,16 +164,18 @@ func (v *defaultInstaller) installDeployment(ctx context.Context) error {
return err
}
-func (v *defaultInstaller) installValidationRules(ctx context.Context) error {
+func (v *defaultInstaller) installAPIServerDeployment(ctx context.Context) error {
timer := time.AfterFunc(
3*time.Second,
- func() { fmt.Fprintln(os.Stderr, color.HiYellowString("WARNING: too long to create Validation rules")) },
+ func() {
+ fmt.Fprintln(os.Stderr, color.HiYellowString("WARNING: too long to create API server Deployment"))
+ },
)
defer timer.Stop()
- err := installValidationRulesDefault(ctx, v.Config)
+ err := installAPIServerDeploymentDefault(ctx, v.Config)
if err != nil && !v.DryRun {
- fmt.Fprintf(os.Stderr, "%v unable to create Validation rules; %v", color.HiRedString("ERROR"), err)
+ fmt.Fprintf(os.Stderr, "%v unable to create API server Deployment; %v", color.HiRedString("ERROR"), err)
}
return err
}
@@ -307,16 +309,18 @@ func (v *defaultInstaller) uninstallDeployment(ctx context.Context) error {
return err
}
-func (v *defaultInstaller) uninstallValidationRules(ctx context.Context) error {
+func (v *defaultInstaller) uninstallAPIServerDeployment(ctx context.Context) error {
timer := time.AfterFunc(
3*time.Second,
- func() { fmt.Fprintln(os.Stderr, color.HiYellowString("WARNING: too long to delete Validation rules")) },
+ func() {
+ fmt.Fprintln(os.Stderr, color.HiYellowString("WARNING: too long to delete API server Deployment"))
+ },
)
defer timer.Stop()
- err := uninstallValidationRulesDefault(ctx, v.Config)
+ err := uninstallAPIServerDeploymentDefault(ctx, v.Config)
if err != nil && !v.DryRun {
- fmt.Fprintf(os.Stderr, "%v unable to delete Validation rules; %v", color.HiRedString("ERROR"), err)
+ fmt.Fprintf(os.Stderr, "%v unable to delete API server Deployment; %v", color.HiRedString("ERROR"), err)
}
return err
}
@@ -349,13 +353,16 @@ func (v *defaultInstaller) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *defaultInstaller) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -365,9 +372,6 @@ func (v *defaultInstaller) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/deployment.go b/pkg/installer/deployment.go
index 392c87770..f92d784e7 100644
--- a/pkg/installer/deployment.go
+++ b/pkg/installer/deployment.go
@@ -27,82 +27,8 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "k8s.io/apimachinery/pkg/util/intstr"
)
-func createControllerSecret(ctx context.Context, publicCertBytes, privateKeyBytes []byte, c *Config) error {
- getCertsDataMap := func() map[string][]byte {
- mp := make(map[string][]byte)
- mp[privateKeyFileName] = privateKeyBytes
- mp[publicCertFileName] = publicCertBytes
- return mp
- }
-
- secret := &corev1.Secret{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Secret",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: admissionWebhookSecretName,
- Namespace: c.namespace(),
- Annotations: defaultAnnotations,
- Labels: defaultLabels,
- },
- Data: getCertsDataMap(),
- }
-
- if c.DryRun {
- return c.postProc(secret)
- }
-
- if _, err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
- if !apierrors.IsAlreadyExists(err) {
- return err
- }
- }
- return c.postProc(secret)
-}
-
-func createControllerService(ctx context.Context, generatedSelectorValue string, c *Config) error {
- admissionWebhookPort := corev1.ServicePort{
- Port: admissionControllerWebhookPort,
- TargetPort: intstr.IntOrString{
- Type: intstr.String,
- StrVal: admissionControllerWebhookName,
- },
- }
- svc := &corev1.Service{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "v1",
- Kind: "Service",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: validationControllerName,
- Namespace: c.namespace(),
- Annotations: defaultAnnotations,
- Labels: defaultLabels,
- },
- Spec: corev1.ServiceSpec{
- Ports: []corev1.ServicePort{admissionWebhookPort},
- Selector: map[string]string{
- selectorKey: generatedSelectorValue,
- },
- },
- }
-
- if c.DryRun {
- return c.postProc(svc)
- }
-
- if _, err := k8s.KubeClient().CoreV1().Services(c.namespace()).Create(ctx, svc, metav1.CreateOptions{}); err != nil {
- if !apierrors.IsAlreadyExists(err) {
- return err
- }
- }
- return c.postProc(svc)
-}
-
func createDeployment(ctx context.Context, c *Config) error {
var replicas int32 = 3
privileged := true
@@ -112,12 +38,6 @@ func createDeployment(ctx context.Context, c *Config) error {
volumeMounts := []corev1.VolumeMount{
newVolumeMount(volumeNameSocketDir, socketDir, corev1.MountPropagationNone, false),
}
-
- if c.AdmissionControl {
- volumes = append(volumes, newSecretVolume(admissionControllerCertsDir, admissionWebhookSecretName))
- volumeMounts = append(volumeMounts, newVolumeMount(admissionControllerCertsDir, admissionCertsDir, corev1.MountPropagationNone, false))
- }
-
podSpec := corev1.PodSpec{
ServiceAccountName: c.serviceAccountName(),
Volumes: volumes,
@@ -163,7 +83,7 @@ func createDeployment(ctx context.Context, c *Config) error {
Args: []string{
"controller",
fmt.Sprintf("-v=%d", logLevel),
- fmt.Sprintf("--identity=%s", c.deploymentName()),
+ fmt.Sprintf("--identity=%s", c.identity()),
fmt.Sprintf("--csi-endpoint=$(%s)", csiEndpointEnvVarName),
fmt.Sprintf("--kube-node-name=$(%s)", kubeNodeNameEnvVarName),
fmt.Sprintf("--readiness-port=%d", consts.ReadinessPort),
@@ -171,11 +91,7 @@ func createDeployment(ctx context.Context, c *Config) error {
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
- Ports: append(commonContainerPorts, corev1.ContainerPort{
- ContainerPort: admissionControllerWebhookPort,
- Name: admissionControllerWebhookName,
- Protocol: corev1.ProtocolTCP,
- }),
+ Ports: commonContainerPorts,
ReadinessProbe: &corev1.Probe{ProbeHandler: getReadinessHandler()},
Env: []corev1.EnvVar{kubeNodeNameEnvVar, csiEndpointEnvVar},
VolumeMounts: volumeMounts,
@@ -183,18 +99,6 @@ func createDeployment(ctx context.Context, c *Config) error {
},
}
- if c.AdmissionControl {
- caCertBytes, publicCertBytes, privateKeyBytes, certErr := getCerts([]string{admissionWehookDNSName})
- if certErr != nil {
- return certErr
- }
- c.validationWebhookCaBundle = caCertBytes
-
- if err := createControllerSecret(ctx, publicCertBytes, privateKeyBytes, c); err != nil {
- return err
- }
- }
-
generatedSelectorValue := generateSanitizedUniqueNameFrom(c.deploymentName())
deployment := &appsv1.Deployment{
@@ -239,11 +143,7 @@ func createDeployment(ctx context.Context, c *Config) error {
}
}
- if err := c.postProc(deployment); err != nil {
- return err
- }
-
- return createControllerService(ctx, generatedSelectorValue, c)
+ return c.postProc(deployment)
}
func installDeploymentDefault(ctx context.Context, c *Config) error {
@@ -251,9 +151,6 @@ func installDeploymentDefault(ctx context.Context, c *Config) error {
}
func uninstallDeploymentDefault(ctx context.Context, c *Config) error {
- if err := k8s.KubeClient().CoreV1().Secrets(c.namespace()).Delete(ctx, admissionWebhookSecretName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
- return err
- }
if err := deleteDeployment(ctx, c.namespace(), c.deploymentName()); err != nil && !apierrors.IsNotFound(err) {
return err
}
diff --git a/pkg/installer/install_test.go b/pkg/installer/install_test.go
index e3f9d91f4..f2c939514 100644
--- a/pkg/installer/install_test.go
+++ b/pkg/installer/install_test.go
@@ -37,7 +37,6 @@ func TestInstaller(t *testing.T) {
ContainerImage: "test-image",
ContainerOrg: "test-org",
ContainerRegistry: "test-registry",
- AdmissionControl: false,
NodeSelector: nil,
Tolerations: nil,
SeccompProfile: "",
diff --git a/pkg/installer/psp.go b/pkg/installer/psp.go
index 9cf3fda26..ec9098235 100644
--- a/pkg/installer/psp.go
+++ b/pkg/installer/psp.go
@@ -52,6 +52,7 @@ func createPodSecurityPolicy(ctx context.Context, i *Config) error {
AllowedHostPaths: []policy.AllowedHostPath{
{PathPrefix: consts.ProcFSDir, ReadOnly: true},
{PathPrefix: consts.SysFSDir, ReadOnly: true},
+ {PathPrefix: consts.UdevDataDir, ReadOnly: true},
{PathPrefix: consts.AppRootDir},
{PathPrefix: socketDir},
{PathPrefix: kubeletDirPath},
diff --git a/pkg/installer/service.go b/pkg/installer/service.go
index 82013327b..bdf381731 100644
--- a/pkg/installer/service.go
+++ b/pkg/installer/service.go
@@ -19,6 +19,7 @@ package installer
import (
"context"
+ "github.com/minio/directpv/pkg/consts"
"github.com/minio/directpv/pkg/k8s"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -26,23 +27,24 @@ import (
)
func installServiceDefault(ctx context.Context, c *Config) error {
- if err := createService(ctx, c); err != nil && !apierrors.IsAlreadyExists(err) {
+ if err := createNodeAPIService(ctx, c); err != nil && !apierrors.IsAlreadyExists(err) {
return err
}
+ // Add more services here..
return nil
}
func uninstallServiceDefault(ctx context.Context, c *Config) error {
- if err := k8s.KubeClient().CoreV1().Services(c.namespace()).Delete(ctx, c.serviceName(), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+ if err := k8s.KubeClient().CoreV1().Services(c.namespace()).Delete(ctx, consts.NodeAPIServerHLSVC, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}
-func createService(ctx context.Context, c *Config) error {
- csiPort := corev1.ServicePort{
- Port: 12345,
- Name: "unused",
+func createNodeAPIService(ctx context.Context, c *Config) error {
+ nodeAPIPort := corev1.ServicePort{
+ Port: consts.NodeAPIPort,
+ Name: consts.NodeAPIPortName,
}
svc := &corev1.Service{
TypeMeta: metav1.TypeMeta{
@@ -50,16 +52,18 @@ func createService(ctx context.Context, c *Config) error {
Kind: "Service",
},
ObjectMeta: metav1.ObjectMeta{
- Name: c.serviceName(),
+ Name: consts.NodeAPIServerHLSVC,
Namespace: c.namespace(),
Annotations: defaultAnnotations,
Labels: defaultLabels,
},
Spec: corev1.ServiceSpec{
- Ports: []corev1.ServicePort{csiPort},
+ Ports: []corev1.ServicePort{nodeAPIPort},
Selector: map[string]string{
serviceSelector: selectorValueEnabled,
},
+ Type: corev1.ServiceTypeClusterIP,
+ ClusterIP: corev1.ClusterIPNone,
},
}
diff --git a/pkg/installer/utils.go b/pkg/installer/utils.go
index 06019155b..4b0c30235 100644
--- a/pkg/installer/utils.go
+++ b/pkg/installer/utils.go
@@ -27,6 +27,7 @@ import (
"github.com/minio/directpv/pkg/k8s"
"github.com/minio/directpv/pkg/types"
corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
@@ -152,3 +153,41 @@ func deleteDeployment(ctx context.Context, identity, name string) error {
}
return nil
}
+
+func createOrUpdateSecret(ctx context.Context, secretName string, dataMap map[string][]byte, c *Config) error {
+ secretsClient := k8s.KubeClient().CoreV1().Secrets(c.namespace())
+ secret := &corev1.Secret{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "v1",
+ Kind: "Secret",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: secretName,
+ Namespace: c.namespace(),
+ Annotations: defaultAnnotations,
+ Labels: defaultLabels,
+ },
+ Data: dataMap,
+ }
+
+ if c.DryRun {
+ return c.postProc(secret)
+ }
+
+ existingSecret, err := secretsClient.Get(ctx, secret.Name, metav1.GetOptions{})
+ if err != nil {
+ if !apierrors.IsNotFound(err) {
+ return err
+ }
+ if _, err := secretsClient.Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ return nil
+ }
+
+ existingSecret.Data = secret.Data
+ if _, err := secretsClient.Update(ctx, existingSecret, metav1.UpdateOptions{}); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/pkg/installer/v1dot18.go b/pkg/installer/v1dot18.go
index 437e318d6..96192639f 100644
--- a/pkg/installer/v1dot18.go
+++ b/pkg/installer/v1dot18.go
@@ -65,8 +65,8 @@ func (v *v1dot18) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot18) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot18) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot18) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot18) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot18) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot18) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot18) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot18) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot18) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/v1dot19.go b/pkg/installer/v1dot19.go
index a61ffb8e9..74c69706a 100644
--- a/pkg/installer/v1dot19.go
+++ b/pkg/installer/v1dot19.go
@@ -65,8 +65,8 @@ func (v *v1dot19) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot19) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot19) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot19) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot19) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot19) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot19) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot19) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot19) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot19) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/v1dot20.go b/pkg/installer/v1dot20.go
index ef98c6fad..9e641c8ee 100644
--- a/pkg/installer/v1dot20.go
+++ b/pkg/installer/v1dot20.go
@@ -65,8 +65,8 @@ func (v *v1dot20) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot20) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot20) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot20) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot20) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot20) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot20) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot20) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot20) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot20) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/v1dot21.go b/pkg/installer/v1dot21.go
index f16053192..533101afe 100644
--- a/pkg/installer/v1dot21.go
+++ b/pkg/installer/v1dot21.go
@@ -65,8 +65,8 @@ func (v *v1dot21) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot21) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot21) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot21) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot21) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot21) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot21) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot21) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot21) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot21) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/v1dot22.go b/pkg/installer/v1dot22.go
index 17c8646d3..962318ec9 100644
--- a/pkg/installer/v1dot22.go
+++ b/pkg/installer/v1dot22.go
@@ -65,8 +65,8 @@ func (v *v1dot22) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot22) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot22) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot22) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot22) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot22) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot22) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot22) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot22) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot22) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/v1dot23.go b/pkg/installer/v1dot23.go
index ab9db8a2f..7aa40d650 100644
--- a/pkg/installer/v1dot23.go
+++ b/pkg/installer/v1dot23.go
@@ -65,8 +65,8 @@ func (v *v1dot23) installDeployment(ctx context.Context) error {
return installDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot23) installValidationRules(ctx context.Context) error {
- return installValidationRulesDefault(ctx, v.Config)
+func (v *v1dot23) installAPIServerDeployment(ctx context.Context) error {
+ return installAPIServerDeploymentDefault(ctx, v.Config)
}
// uninstallers
@@ -106,8 +106,8 @@ func (v *v1dot23) uninstallDeployment(ctx context.Context) error {
return uninstallDeploymentDefault(ctx, v.Config)
}
-func (v *v1dot23) uninstallValidationRules(ctx context.Context) error {
- return uninstallValidationRulesDefault(ctx, v.Config)
+func (v *v1dot23) uninstallAPIServerDeployment(ctx context.Context) error {
+ return uninstallAPIServerDeploymentDefault(ctx, v.Config)
}
func (v *v1dot23) Install(ctx context.Context) error {
@@ -138,13 +138,16 @@ func (v *v1dot23) Install(ctx context.Context) error {
if err := v.installDeployment(ctx); err != nil {
return err
}
- return v.installValidationRules(ctx)
+ return v.installAPIServerDeployment(ctx)
}
func (v *v1dot23) Uninstall(ctx context.Context) error {
if err := v.uninstallCRD(ctx); err != nil {
return err
}
+ if err := v.uninstallAPIServerDeployment(ctx); err != nil {
+ return err
+ }
if err := v.uninstallDeployment(ctx); err != nil {
return err
}
@@ -154,9 +157,6 @@ func (v *v1dot23) Uninstall(ctx context.Context) error {
if err := v.uninstallService(ctx); err != nil {
return err
}
- if err := v.uninstallValidationRules(ctx); err != nil {
- return err
- }
if err := v.uninstallStorageClass(ctx); err != nil {
return err
}
diff --git a/pkg/installer/validationrules.go b/pkg/installer/validationrules.go
deleted file mode 100644
index 536ad8e97..000000000
--- a/pkg/installer/validationrules.go
+++ /dev/null
@@ -1,145 +0,0 @@
-// This file is part of MinIO DirectPV
-// Copyright (c) 2021, 2022 MinIO, Inc.
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see .
-
-package installer
-
-import (
- "context"
-
- "github.com/minio/directpv/pkg/consts"
- "github.com/minio/directpv/pkg/k8s"
- admissionv1 "k8s.io/api/admissionregistration/v1"
- apierrors "k8s.io/apimachinery/pkg/api/errors"
- metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-func registerDriveValidationRules(ctx context.Context, c *Config) error {
- driveValidatingWebhookConfig := getDriveValidatingWebhookConfig(c)
- if !c.DryRun {
- if _, err := k8s.KubeClient().AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(ctx, &driveValidatingWebhookConfig, metav1.CreateOptions{}); err != nil {
- if !apierrors.IsAlreadyExists(err) {
- return err
- }
- }
- }
- return c.postProc(driveValidatingWebhookConfig)
-}
-
-func getDriveValidatingWebhookConfig(c *Config) admissionv1.ValidatingWebhookConfiguration {
- getServiceRef := func() *admissionv1.ServiceReference {
- path := "/validatedrive"
- return &admissionv1.ServiceReference{
- Namespace: c.namespace(),
- Name: validationControllerName,
- Path: &path,
- }
- }
-
- getClientConfig := func() admissionv1.WebhookClientConfig {
- return admissionv1.WebhookClientConfig{
- Service: getServiceRef(),
- CABundle: c.validationWebhookCaBundle,
- }
- }
-
- getValidationRules := func() []admissionv1.RuleWithOperations {
- return []admissionv1.RuleWithOperations{
- {
- Operations: []admissionv1.OperationType{admissionv1.Update},
- Rule: admissionv1.Rule{
- APIGroups: []string{"*"},
- APIVersions: []string{"*"},
- Resources: []string{consts.DriveResource},
- },
- },
- }
- }
-
- getValidatingWebhooks := func() []admissionv1.ValidatingWebhook {
- supportedReviewVersions := []string{"v1", "v1beta1", "v1beta2", "v1beta3"}
- sideEffectClass := admissionv1.SideEffectClassNone
- return []admissionv1.ValidatingWebhook{
- {
- Name: validationWebhookConfigName,
- ClientConfig: getClientConfig(),
- AdmissionReviewVersions: supportedReviewVersions,
- SideEffects: &sideEffectClass,
- Rules: getValidationRules(),
- },
- }
- }
-
- validatingWebhookConfiguration := admissionv1.ValidatingWebhookConfiguration{
- TypeMeta: metav1.TypeMeta{
- APIVersion: "admissionregistration.k8s.io/v1",
- Kind: "ValidatingWebhookConfiguration",
- },
- ObjectMeta: metav1.ObjectMeta{
- Name: validationWebhookConfigName,
- Namespace: c.namespace(),
- Annotations: defaultAnnotations,
- Labels: defaultLabels,
- Finalizers: []string{c.namespace() + deleteProtectionFinalizer},
- },
- Webhooks: getValidatingWebhooks(),
- }
-
- return validatingWebhookConfiguration
-}
-
-func deleteDriveValidationRules(ctx context.Context, c *Config) error {
- vClient := k8s.KubeClient().AdmissionregistrationV1().ValidatingWebhookConfigurations()
-
- getDeleteProtectionFinalizer := func() string {
- return c.namespace() + deleteProtectionFinalizer
- }
-
- clearFinalizers := func() error {
- config, err := vClient.Get(ctx, validationWebhookConfigName, metav1.GetOptions{})
- if err != nil {
- return err
- }
- finalizer := getDeleteProtectionFinalizer()
- config.SetFinalizers(k8s.RemoveFinalizer(&config.ObjectMeta, finalizer))
- if _, err := vClient.Update(ctx, config, metav1.UpdateOptions{}); err != nil {
- return err
- }
- return nil
- }
-
- if err := clearFinalizers(); err != nil {
- return err
- }
-
- if err := vClient.Delete(ctx, validationWebhookConfigName, metav1.DeleteOptions{}); err != nil {
- return err
- }
- return nil
-}
-
-func installValidationRulesDefault(ctx context.Context, c *Config) error {
- if !c.AdmissionControl {
- return nil
- }
- return registerDriveValidationRules(ctx, c)
-}
-
-func uninstallValidationRulesDefault(ctx context.Context, c *Config) error {
- if err := deleteDriveValidationRules(ctx, c); err != nil && !apierrors.IsNotFound(err) {
- return err
- }
- return nil
-}
diff --git a/pkg/metrics/collector.go b/pkg/metrics/collector.go
index c822c33fe..e3bb8661d 100644
--- a/pkg/metrics/collector.go
+++ b/pkg/metrics/collector.go
@@ -21,8 +21,8 @@ import (
"github.com/minio/directpv/pkg/client"
"github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/device"
"github.com/minio/directpv/pkg/k8s"
- "github.com/minio/directpv/pkg/sys"
"github.com/minio/directpv/pkg/types"
"github.com/minio/directpv/pkg/volume"
"github.com/minio/directpv/pkg/xfs"
@@ -44,7 +44,7 @@ func newMetricsCollector(nodeID string) *metricsCollector {
return &metricsCollector{
nodeID: nodeID,
desc: prometheus.NewDesc(consts.AppName+"_stats", "Statistics exposed by "+consts.AppPrettyName, nil, nil),
- getDeviceByFSUUID: sys.GetDeviceByFSUUID,
+ getDeviceByFSUUID: device.GetDeviceByFSUUID,
getQuota: xfs.GetQuota,
}
}
diff --git a/pkg/node/server.go b/pkg/node/server.go
index 43f1780dd..07f470617 100644
--- a/pkg/node/server.go
+++ b/pkg/node/server.go
@@ -22,6 +22,7 @@ import (
"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/minio/directpv/pkg/client"
+ "github.com/minio/directpv/pkg/device"
"github.com/minio/directpv/pkg/metrics"
"github.com/minio/directpv/pkg/sys"
"github.com/minio/directpv/pkg/types"
@@ -53,7 +54,7 @@ type Server struct {
// NewServer creates node server.
func NewServer(ctx context.Context,
identity, nodeID, rack, zone, region string,
- reflinkSupport bool, metricsPort int,
+ metricsPort int,
) (*Server, error) {
nodeServer := &Server{
nodeID: nodeID,
@@ -63,7 +64,7 @@ func NewServer(ctx context.Context,
region: region,
getMounts: sys.GetMounts,
- getDeviceByFSUUID: sys.GetDeviceByFSUUID,
+ getDeviceByFSUUID: device.GetDeviceByFSUUID,
bindMount: xfs.BindMount,
unmount: func(target string) error { return sys.Unmount(target, true, true, false) },
getQuota: xfs.GetQuota,
diff --git a/pkg/rest/api-response.go b/pkg/rest/api-response.go
new file mode 100644
index 000000000..12e9f5512
--- /dev/null
+++ b/pkg/rest/api-response.go
@@ -0,0 +1,62 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "encoding/json"
+ "net/http"
+ "strconv"
+
+ "k8s.io/klog/v2"
+)
+
+// writeSuccessResponseJSON writes success headers and response if any,
+// with content-type set to `application/json`.
+func writeSuccessResponse(w http.ResponseWriter, response []byte) {
+ writeResponse(w, http.StatusOK, response)
+}
+
+// writeSuccessResponse writes error response with the provided statusCode
+// with content-type set to `application/json`.
+func writeErrorResponse(w http.ResponseWriter, statusCode int, apiErr apiError) {
+ if statusCode == 0 {
+ statusCode = http.StatusInternalServerError
+ }
+ var responseBytes []byte
+ var err error
+ responseBytes, err = json.Marshal(apiErr)
+ if err != nil {
+ klog.Errorf("couldn't marshal the apiError %v: %v", apiErr, err)
+ responseBytes = []byte(apiErr.Description)
+ }
+ writeResponse(w, statusCode, responseBytes)
+}
+
+// writeResponse writes the response bytes to the response writer
+// with content-type set to `application/json`
+func writeResponse(w http.ResponseWriter, statusCode int, response []byte) {
+ if statusCode == 0 || statusCode < 100 || statusCode > 999 {
+ klog.Errorf("invalid WriteHeader code %v", statusCode)
+ statusCode = http.StatusInternalServerError
+ }
+ w.Header().Add("Content-Type", "application/json")
+ w.Header().Set("Content-Length", strconv.Itoa(len(response)))
+ w.WriteHeader(statusCode)
+ if response != nil {
+ w.Write(response)
+ }
+}
diff --git a/pkg/rest/api.go b/pkg/rest/api.go
new file mode 100644
index 000000000..1ca6df8ca
--- /dev/null
+++ b/pkg/rest/api.go
@@ -0,0 +1,321 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "path"
+ "sync"
+
+ "github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/ellipsis"
+ "github.com/minio/directpv/pkg/k8s"
+ corev1 "k8s.io/api/core/v1"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+)
+
+const (
+ devicesListAPIPath = "/devices/list"
+ devicesFormatAPIPath = "/devices/format"
+)
+
+var (
+ apiServerPrivateKeyPath = path.Join(consts.APIServerCertsPath, consts.PrivateKeyFileName)
+ apiServerCertPath = path.Join(consts.APIServerCertsPath, consts.PublicCertFileName)
+)
+
+// ServeAPIServer starts the API server
+func ServeAPIServer(ctx context.Context, apiPort int) error {
+ certs, err := tls.LoadX509KeyPair(apiServerCertPath, apiServerPrivateKeyPath)
+ if err != nil {
+ klog.Errorf("Filed to load key pair for the DirectPV API server: %v", err)
+ return err
+ }
+ // Create a secure http server
+ server := &http.Server{
+ TLSConfig: &tls.Config{
+ Certificates: []tls.Certificate{certs},
+ InsecureSkipVerify: true,
+ },
+ // TODO: Implement GetCertificate
+ }
+
+ // define http server and server handler
+ mux := http.NewServeMux()
+ mux.HandleFunc(devicesListAPIPath, listDevicesHandler)
+ mux.HandleFunc(devicesFormatAPIPath, formatDrivesHandler)
+ mux.HandleFunc(consts.ReadinessPath, readinessHandler)
+ server.Handler = mux
+
+ lc := net.ListenConfig{}
+ listener, lErr := lc.Listen(ctx, "tcp", fmt.Sprintf(":%v", apiPort))
+ if lErr != nil {
+ return lErr
+ }
+
+ errCh := make(chan error)
+ go func() {
+ klog.V(3).Infof("Starting API server in port: %d", apiPort)
+ if err := server.ServeTLS(listener, "", ""); err != nil {
+ klog.Errorf("Failed to listen and serve API server: %v", err)
+ errCh <- err
+ }
+ }()
+
+ return <-errCh
+}
+
+// listDevicesHandler gathers the list of available and unavailable devices from the nodes
+func listDevicesHandler(w http.ResponseWriter, r *http.Request) {
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ klog.Errorf("couldn't read the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't read the request"))
+ return
+ }
+ var req GetDevicesRequest
+ if err = json.Unmarshal(data, &req); err != nil {
+ klog.Errorf("couldn't parse the request: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't parse the request"))
+ return
+ }
+ deviceInfo, err := listDevices(context.Background(), req)
+ if err != nil {
+ klog.Errorf("couldn't get the drive list: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't get the drive list"))
+ return
+ }
+ jsonBytes, err := json.Marshal(GetDevicesResponse{
+ DeviceInfo: deviceInfo,
+ })
+ if err != nil {
+ klog.Errorf("couldn't marshal the format status: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't marshal the format status"))
+ return
+ }
+ writeSuccessResponse(w, jsonBytes)
+}
+
+// listDevices queries the nodes parallelly to get the available and unavailable devices
+func listDevices(ctx context.Context, req GetDevicesRequest) (map[NodeName][]Device, error) {
+ var nodes []string
+ var err error
+ if len(req.Nodes) > 0 {
+ nodes, err = ellipsis.Expand(string(req.Nodes))
+ if err != nil {
+ return nil, fmt.Errorf("couldn't expand the node selector %v: %v", req.Nodes, err)
+ }
+ }
+ endpointsMap, nodeAPIPort, err := getNodeEndpoints(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't get the node endpoints: %v", err)
+ }
+ httpClient := &http.Client{
+ Transport: getTransport(),
+ }
+ reqBody, err := json.Marshal(GetDevicesRequest{
+ Drives: req.Drives,
+ Statuses: req.Statuses,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("errror while marshalling the request: %v", err)
+ }
+ var devices = make(map[NodeName][]Device)
+ var mutex = &sync.RWMutex{}
+ var wg sync.WaitGroup
+ for node, ip := range endpointsMap {
+ if len(nodes) > 0 && !stringIn(nodes, node) {
+ continue
+ }
+ wg.Add(1)
+ go func(node, ip string) {
+ defer wg.Done()
+ reqURL := fmt.Sprintf("https://%s:%d%s", ip, nodeAPIPort, devicesListAPIPath)
+ req, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(reqBody))
+ if err != nil {
+ klog.Infof("error while constructing request: %v", err)
+ return
+ }
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ klog.Errorf("failed to get the result from node: %s, url: %s, error: %v", node, req.URL, err)
+ return
+ }
+ defer drainBody(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ klog.Errorf("failed to get the result from node: %s, url: %s, statusCode: %d", node, req.URL, resp.StatusCode)
+ return
+ }
+ nodeResponseInBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ klog.Errorf("failed to read response from node: %s, url: %s: %v", node, req.URL, err)
+ return
+ }
+ var nodeResponse GetDevicesResponse
+ if err := json.Unmarshal(nodeResponseInBytes, &nodeResponse); err != nil {
+ klog.Errorf("couldn't parse the response from node: %s, url: %s: %v", node, req.URL, err)
+ return
+ }
+ for k, v := range nodeResponse.DeviceInfo {
+ mutex.Lock()
+ devices[k] = v
+ mutex.Unlock()
+ }
+ }(node, ip)
+ }
+ wg.Wait()
+ return devices, nil
+}
+
+// formatDrivesHandler forwards the format requests to respective nodes
+func formatDrivesHandler(w http.ResponseWriter, r *http.Request) {
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ klog.Errorf("couldn't read the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't read the request"))
+ return
+ }
+ var req FormatDevicesRequest
+ if err = json.Unmarshal(data, &req); err != nil {
+ klog.Errorf("couldn't parse the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't parse the request"))
+ return
+ }
+ formatStatus, err := formatDrives(context.Background(), req)
+ if err != nil {
+ klog.Errorf("couldn't format the drives: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't format the drives"))
+ return
+ }
+ // Marshal API response
+ jsonBytes, err := json.Marshal(FormatDevicesResponse{
+ DeviceInfo: formatStatus,
+ })
+ if err != nil {
+ klog.Errorf("Couldn't marshal the format status: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't format the drives"))
+ return
+ }
+ writeSuccessResponse(w, jsonBytes)
+}
+
+// formatDrives forwards the format requests to respective nodes
+func formatDrives(ctx context.Context, req FormatDevicesRequest) (map[NodeName][]FormatDeviceStatus, error) {
+ endpointsMap, nodeAPIPort, err := getNodeEndpoints(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't get the node endpoints: %v", err)
+ }
+ httpClient := &http.Client{
+ Transport: getTransport(),
+ }
+ var wg sync.WaitGroup
+ var formatStatus = make(map[NodeName][]FormatDeviceStatus)
+ var mutex = &sync.RWMutex{}
+ for node, formatDevices := range req.FormatInfo {
+ endpoint, ok := endpointsMap[string(node)]
+ if !ok {
+ klog.Errorf("couldn't find an endpoint for %s", node)
+ continue
+ }
+ wg.Add(1)
+ go func(node NodeName, ip string, formatDevices []FormatDevice) {
+ defer wg.Done()
+ reqBody, err := json.Marshal(FormatDevicesRequest{
+ FormatInfo: map[NodeName][]FormatDevice{
+ node: formatDevices,
+ },
+ })
+ if err != nil {
+ klog.Infof("error while parsing format devices request: %v", err)
+ return
+ }
+ reqURL := fmt.Sprintf("https://%s:%d%s", ip, nodeAPIPort, devicesFormatAPIPath)
+ req, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(reqBody))
+ if err != nil {
+ klog.Infof("error while constructing request: %v", err)
+ return
+ }
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ klog.Errorf("failed to get the result from node: %s, url: %s, error: %v", node, req.URL, err)
+ return
+ }
+ defer drainBody(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ klog.Errorf("failed to get the result from node: %s, url: %s, statusCode: %d", node, req.URL, resp.StatusCode)
+ return
+ }
+ nodeResponseInBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ klog.Errorf("failed to read response from node: %s, url: %s: %v", node, req.URL, err)
+ return
+ }
+ var nodeResponse FormatDevicesResponse
+ if err = json.Unmarshal(nodeResponseInBytes, &nodeResponse); err != nil {
+ klog.Errorf("couldn't parse the response from node: %s, url: %s: %v", node, req.URL, err)
+ return
+ }
+ for k, v := range nodeResponse.DeviceInfo {
+ mutex.Lock()
+ formatStatus[k] = v
+ mutex.Unlock()
+ }
+ }(node, endpoint, formatDevices)
+ }
+ wg.Wait()
+ return formatStatus, nil
+}
+
+// getNodeEndpoints reads the endpoint objects present in the node svc to get the endpoints of the nodes
+func getNodeEndpoints(ctx context.Context) (endpointsMap map[string]string, apiPort int, err error) {
+ var endpoints *corev1.Endpoints
+ endpoints, err = k8s.KubeClient().CoreV1().Endpoints(consts.Namespace).Get(ctx, consts.NodeAPIServerHLSVC, metav1.GetOptions{})
+ if err != nil {
+ return
+ }
+ if len(endpoints.Subsets) == 0 {
+ err = errNoSubsetsFound
+ return
+ }
+ endpointsMap = make(map[string]string)
+ for _, address := range endpoints.Subsets[0].Addresses {
+ endpointsMap[*address.NodeName] = address.IP
+ }
+ if len(endpointsMap) == 0 {
+ err = errNoEndpointsFound
+ return
+ }
+ for _, port := range endpoints.Subsets[0].Ports {
+ if port.Name == consts.NodeAPIPortName {
+ apiPort = int(port.Port)
+ break
+ }
+ }
+ if apiPort == 0 {
+ err = errNodeAPIPortNotFound
+ }
+ return
+}
diff --git a/pkg/rest/errors.go b/pkg/rest/errors.go
new file mode 100644
index 000000000..b354e8857
--- /dev/null
+++ b/pkg/rest/errors.go
@@ -0,0 +1,41 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import "errors"
+
+var (
+ errNoSubsetsFound = errors.New("no subsets found for the node service")
+ errNoEndpointsFound = errors.New("no endpoints found for the node service")
+ errNodeAPIPortNotFound = errors.New("api port for the node endpoints not found")
+ errMountFailure = errors.New("could not mount the drive")
+ errUDevDataMismatch = errors.New("udev data isn't matching")
+ errForceRequired = errors.New("force flag is required for formatting")
+ errDuplicateDevice = errors.New("found duplicate devices for drive")
+)
+
+type apiError struct {
+ Description string `json:"description,omitempty"`
+ Message string `json:"message"`
+}
+
+func toAPIError(err error, message string) apiError {
+ return apiError{
+ Description: err.Error(),
+ Message: message,
+ }
+}
diff --git a/pkg/rest/node-handlers.go b/pkg/rest/node-handlers.go
new file mode 100644
index 000000000..4661db2d1
--- /dev/null
+++ b/pkg/rest/node-handlers.go
@@ -0,0 +1,151 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "context"
+ "errors"
+ "os"
+ "sync"
+
+ "github.com/google/uuid"
+ "github.com/minio/directpv/pkg/device"
+ "github.com/minio/directpv/pkg/sys"
+ "github.com/minio/directpv/pkg/types"
+ "github.com/minio/directpv/pkg/xfs"
+ losetup "gopkg.in/freddierice/go-losetup.v1"
+
+ "k8s.io/klog/v2"
+)
+
+// nodeAPIHandlers provides HTTP handlers for DirectPV node API.
+type nodeAPIHandler struct {
+ nodeID string
+ reflinkSupport bool
+ topology map[string]string
+ mountDevice func(device, target string) error
+ makeFS func(ctx context.Context, device, uuid string, force, reflink bool) error
+ safeUnmount func(target string, force, detach, expire bool) error
+ truncate func(name string, size int64) error
+ attachLoopDevice func(backingFile string, offset uint64, ro bool) (losetup.Device, error)
+ readRunUdevDataByMajorMinor func(majorMinor string) (map[string]string, error)
+ probeXFS func(path string) (fsuuid, label string, totalCapacity, freeCapacity uint64, err error)
+ // locks
+ formatLockerMutex sync.Mutex
+ formatLocker map[string]*sync.Mutex
+}
+
+// Unmount(target string, force, detach, expire bool) error {
+func newNodeAPIHandler(ctx context.Context, identity, nodeID, rack, zone, region string) (*nodeAPIHandler, error) {
+ var err error
+ nodeAPIHandler := &nodeAPIHandler{
+ nodeID: nodeID,
+ mountDevice: xfs.Mount,
+ makeFS: xfs.MakeFS,
+ safeUnmount: sys.Unmount,
+ truncate: os.Truncate,
+ attachLoopDevice: losetup.Attach,
+ readRunUdevDataByMajorMinor: device.ReadRunUdevDataByMajorMinor,
+ probeXFS: xfs.Probe,
+ formatLocker: map[string]*sync.Mutex{},
+ topology: map[string]string{
+ string(types.TopologyDriverIdentity): identity,
+ string(types.TopologyDriverRack): rack,
+ string(types.TopologyDriverZone): zone,
+ string(types.TopologyDriverRegion): region,
+ string(types.TopologyDriverNode): nodeID,
+ },
+ }
+ nodeAPIHandler.reflinkSupport, err = nodeAPIHandler.isReflinkSupported(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return nodeAPIHandler, nil
+}
+
+func (n *nodeAPIHandler) getFormatLock(majorMinor string) *sync.Mutex {
+ n.formatLockerMutex.Lock()
+ defer n.formatLockerMutex.Unlock()
+
+ if _, found := n.formatLocker[majorMinor]; !found {
+ n.formatLocker[majorMinor] = &sync.Mutex{}
+ }
+
+ return n.formatLocker[majorMinor]
+}
+
+func (n *nodeAPIHandler) isReflinkSupported(ctx context.Context) (bool, error) {
+ var reflinkSupport bool
+ // trying with reflink enabled
+ if err := n.checkXFS(ctx, true); err == nil {
+ reflinkSupport = true
+ klog.V(3).Infof("enabled reflink while formatting")
+ } else {
+ if !errors.Is(err, errMountFailure) {
+ return reflinkSupport, err
+ }
+ // trying with reflink disabled
+ if err := n.checkXFS(ctx, false); err != nil {
+ return reflinkSupport, err
+ }
+ reflinkSupport = false
+ klog.V(3).Infof("disabled reflink while formatting")
+ }
+ return reflinkSupport, nil
+}
+
+func (n *nodeAPIHandler) checkXFS(ctx context.Context, reflinkSupport bool) error {
+ mountPoint, err := os.MkdirTemp("", "xfs.check.mnt.")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(mountPoint)
+
+ file, err := os.CreateTemp("", "xfs.check.file.")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(file.Name())
+ file.Close()
+
+ if err = n.truncate(file.Name(), xfs.MinSupportedDeviceSize); err != nil {
+ return err
+ }
+
+ if err = n.makeFS(ctx, file.Name(), uuid.New().String(), false, reflinkSupport); err != nil {
+ klog.V(3).ErrorS(err, "failed to format", "reflink", reflinkSupport)
+ return err
+ }
+
+ loopDevice, err := n.attachLoopDevice(file.Name(), 0, false)
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ if err := loopDevice.Detach(); err != nil {
+ klog.Error(err)
+ }
+ }()
+
+ if err = n.mountDevice(loopDevice.Path(), mountPoint); err != nil {
+ klog.V(3).ErrorS(err, "failed to mount", "reflink", reflinkSupport)
+ return errMountFailure
+ }
+
+ return n.safeUnmount(mountPoint, true, true, false)
+}
diff --git a/pkg/rest/node.go b/pkg/rest/node.go
new file mode 100644
index 000000000..78c194e89
--- /dev/null
+++ b/pkg/rest/node.go
@@ -0,0 +1,397 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "path"
+ "reflect"
+ "sync"
+
+ "github.com/google/uuid"
+ "github.com/hashicorp/errwrap"
+ apiTypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
+ "github.com/minio/directpv/pkg/client"
+ "github.com/minio/directpv/pkg/consts"
+ "github.com/minio/directpv/pkg/device"
+ "github.com/minio/directpv/pkg/drive"
+ "github.com/minio/directpv/pkg/ellipsis"
+ "github.com/minio/directpv/pkg/types"
+
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/klog/v2"
+)
+
+var (
+ nodeAPIServerPrivateKeyPath = path.Join(consts.NodeAPIServerCertsPath, consts.PrivateKeyFileName)
+ nodeAPIServerCertPath = path.Join(consts.NodeAPIServerCertsPath, consts.PublicCertFileName)
+)
+
+// suggestions
+var (
+ formatRetrySuggestion = "retry the format request"
+ formatRetryWithForceSuggestion = "retry the format request with force"
+)
+
+// reasons
+var (
+ udevDataMismatchReason = "probed udevdata isn't matching with the udev data in the request"
+ metaDataPathSuffix = path.Join(fmt.Sprintf(".%s.sys", consts.AppName), "metadata.json")
+)
+
+// ServeNodeAPIServer starts the DirectPV Node API server
+func ServeNodeAPIServer(ctx context.Context, nodeAPIPort int, identity, nodeID, rack, zone, region string) error {
+ certs, err := tls.LoadX509KeyPair(nodeAPIServerCertPath, nodeAPIServerPrivateKeyPath)
+ if err != nil {
+ klog.Errorf("Filed to load key pair: %v", err)
+ return err
+ }
+
+ // Create a secure http server
+ server := &http.Server{
+ TLSConfig: &tls.Config{
+ Certificates: []tls.Certificate{certs},
+ InsecureSkipVerify: true,
+ },
+ }
+
+ nodeHandler, err := newNodeAPIHandler(ctx, identity, nodeID, rack, zone, region)
+ if err != nil {
+ return err
+ }
+
+ // define http server and server handler
+ mux := http.NewServeMux()
+ mux.HandleFunc(devicesListAPIPath, nodeHandler.listLocalDevicesHandler)
+ mux.HandleFunc(devicesFormatAPIPath, nodeHandler.formatLocalDevicesHandler)
+ mux.HandleFunc(consts.ReadinessPath, readinessHandler)
+ server.Handler = mux
+
+ lc := net.ListenConfig{}
+ listener, lErr := lc.Listen(ctx, "tcp", fmt.Sprintf(":%v", nodeAPIPort))
+ if lErr != nil {
+ return lErr
+ }
+
+ errCh := make(chan error)
+ go func() {
+ klog.V(3).Infof("Starting DirectPV Node API server in port: %d", nodeAPIPort)
+ if err := server.ServeTLS(listener, "", ""); err != nil {
+ klog.Errorf("Failed to listen and serve DirectPV Node API server: %v", err)
+ errCh <- err
+ }
+ }()
+
+ return <-errCh
+}
+
+// listLocalDevicesHandler fetches the devices present in the node and sends back
+func (n *nodeAPIHandler) listLocalDevicesHandler(w http.ResponseWriter, r *http.Request) {
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ klog.Errorf("couldn't read the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't read the request"))
+ return
+ }
+ // Unmarshal API request
+ var req GetDevicesRequest
+ if err = json.Unmarshal(data, &req); err != nil {
+ klog.Errorf("couldn't parse the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't parse the request"))
+ return
+ }
+ deviceList, err := n.listLocalDevices(context.Background(), req)
+ if err != nil {
+ klog.Errorf("couldn't list local drives: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't list local drives"))
+ return
+ }
+ jsonBytes, err := json.Marshal(GetDevicesResponse{
+ DeviceInfo: map[NodeName][]Device{
+ NodeName(n.nodeID): deviceList,
+ },
+ })
+ if err != nil {
+ klog.Errorf("Couldn't marshal the response: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't marshal the response"))
+ return
+ }
+ writeSuccessResponse(w, jsonBytes)
+}
+
+func (n *nodeAPIHandler) listLocalDevices(ctx context.Context, req GetDevicesRequest) ([]Device, error) {
+ var driveSelectors, statusSelectors []string
+ var err error
+ if len(req.Drives) > 0 {
+ driveSelectors, err = ellipsis.Expand(string(req.Drives))
+ if err != nil {
+ return nil, fmt.Errorf("couldn't expand the node selector %v: %v", req.Nodes, err)
+ }
+ }
+ for _, status := range req.Statuses {
+ statusSelectors = append(statusSelectors, string(status))
+ }
+ // Probe the devices from the node
+ devices, err := device.ProbeDevices()
+ if err != nil {
+ return nil, fmt.Errorf("couldn't probe the devices: %v", err)
+ }
+ // Fetch the local drives from the k8s
+ drives, err := n.listDrives(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't fetch the drives: %v", err)
+ }
+ var deviceList []Device
+ for _, drive := range drives {
+ matchedDevices, unmatchedDevices := getMatchedDevicesForDrive(&drive, devices)
+ switch len(matchedDevices) {
+ case 0:
+ // Drive which was online before is lost/detached/corrupted now
+ if len(statusSelectors) > 0 && !stringIn(statusSelectors, string(DeviceStatusUnavailable)) {
+ break
+ }
+ deviceName := path.Base(drive.Status.Path)
+ if len(driveSelectors) > 0 && !stringIn(driveSelectors, deviceName) {
+ break
+ }
+ deviceList = append(deviceList, Device{
+ Name: deviceName,
+ Size: uint64(drive.Status.TotalCapacity),
+ Model: drive.Status.ModelNumber,
+ Vendor: drive.Status.Vendor,
+ Filesystem: "xfs",
+ Status: DeviceStatusUnavailable,
+ Description: "corrupted/lost drive",
+ })
+ case 1:
+ // Drive detected
+ if len(statusSelectors) > 0 && !stringIn(statusSelectors, string(DeviceStatusUnavailable)) {
+ break
+ }
+ if len(driveSelectors) > 0 && !stringIn(driveSelectors, matchedDevices[0].Name) {
+ break
+ }
+ deviceList = append(deviceList, Device{
+ Name: matchedDevices[0].Name,
+ MajorMinor: matchedDevices[0].MajorMinor,
+ Size: matchedDevices[0].Size,
+ Model: matchedDevices[0].Model(),
+ Vendor: matchedDevices[0].Vendor(),
+ Filesystem: matchedDevices[0].FSType(),
+ Status: DeviceStatusUnavailable,
+ Description: "formatted drive",
+ UDevData: matchedDevices[0].UDevData,
+ })
+ default:
+ // Multiple matches found for the Online drive
+ klog.ErrorS(errDuplicateDevice, "drive: ", drive.Name, " devices: ", getDeviceNames(matchedDevices))
+ }
+ devices = unmatchedDevices
+ }
+ for _, device := range devices {
+ deviceStatus := DeviceStatusAvailable
+ isUnavailable, description := device.IsUnavailable()
+ if isUnavailable {
+ deviceStatus = DeviceStatusUnavailable
+ }
+ if len(statusSelectors) > 0 && !stringIn(statusSelectors, string(deviceStatus)) {
+ continue
+ }
+ if len(driveSelectors) > 0 && !stringIn(driveSelectors, device.Name) {
+ continue
+ }
+ deviceList = append(deviceList, Device{
+ Name: device.Name,
+ MajorMinor: device.MajorMinor,
+ Size: device.Size,
+ Model: device.Model(),
+ Vendor: device.Vendor(),
+ Filesystem: device.FSType(),
+ Status: deviceStatus,
+ Description: description,
+ UDevData: device.UDevData,
+ })
+ }
+ return deviceList, nil
+}
+
+func (n *nodeAPIHandler) listDrives(ctx context.Context) ([]types.Drive, error) {
+ labelSelector := fmt.Sprintf("%s=%s", types.NodeLabelKey, types.NewLabelValue(n.nodeID))
+ result, err := client.DriveClient().List(ctx, metav1.ListOptions{
+ LabelSelector: labelSelector,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return result.Items, nil
+}
+
+// formatLocalDevicesHandler formats the devices present in the node and returns back the status
+func (n *nodeAPIHandler) formatLocalDevicesHandler(w http.ResponseWriter, r *http.Request) {
+ data, err := io.ReadAll(r.Body)
+ if err != nil {
+ klog.Errorf("couldn't read the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't read the request"))
+ return
+ }
+ var req FormatDevicesRequest
+ if err = json.Unmarshal(data, &req); err != nil {
+ klog.Errorf("couldn't parse the request: %v", err)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "couldn't parse the request"))
+ return
+ }
+ formatDevices, ok := req.FormatInfo[NodeName(n.nodeID)]
+ if !ok {
+ klog.Errorf("nodename not found in the request. expected %s", n.nodeID)
+ writeErrorResponse(w, http.StatusBadRequest, toAPIError(err, "nodename not found in the request"))
+ return
+ }
+ var formatStatusList []FormatDeviceStatus
+ var wg sync.WaitGroup
+ for _, formatDevice := range formatDevices {
+ wg.Add(1)
+ go func(device FormatDevice) {
+ defer wg.Done()
+ formatStatus := n.format(context.Background(), device)
+ if formatStatus.Error == "" {
+ if err := n.addDrive(context.Background(), device, formatStatus); err != nil {
+ klog.Errorf("failed to create a new drive %s for device %s; %w", formatStatus.FSUUID, device.Name, err)
+ formatStatus.setErr(err, "failed to create a new drive", formatRetrySuggestion)
+ }
+ }
+ // Incase of error, umount the target so that the request can be retried
+ if formatStatus.Error != "" && formatStatus.mountedAt != "" {
+ if err := n.safeUnmount(formatStatus.mountedAt, false, false, false); err != nil {
+ formatStatus.setErr(
+ errwrap.Wrap(err, errors.New(formatStatus.Error)),
+ "failed to umount on failure",
+ fmt.Sprintf("please umount %s and retry the format request", formatStatus.mountedAt),
+ )
+ }
+ }
+ formatStatusList = append(formatStatusList, formatStatus)
+ }(formatDevice)
+ }
+ wg.Wait()
+ // Marshal API response
+ jsonBytes, err := json.Marshal(FormatDevicesResponse{
+ DeviceInfo: map[NodeName][]FormatDeviceStatus{
+ NodeName(n.nodeID): formatStatusList,
+ },
+ })
+ if err != nil {
+ klog.Errorf("Couldn't marshal the format status: %v", err)
+ writeErrorResponse(w, http.StatusInternalServerError, toAPIError(err, "couldn't marshal the format status"))
+ return
+ }
+ writeSuccessResponse(w, jsonBytes)
+}
+
+func (n *nodeAPIHandler) format(ctx context.Context, device FormatDevice) (formatStatus FormatDeviceStatus) {
+ var totalCapacity, freeCapacity uint64
+ // Get format lock
+ n.getFormatLock(device.MajorMinor).Lock()
+ defer n.getFormatLock(device.MajorMinor).Unlock()
+ formatStatus.Name = device.Name
+ // Check if the udev data is matching
+ udevData, err := n.readRunUdevDataByMajorMinor(device.MajorMinor)
+ if err != nil {
+ klog.V(3).Infof("error while reading udevdata for device %s: %v", device.Name, err)
+ formatStatus.setErr(err, "couldn't read the udev data", "")
+ return
+ }
+ if !reflect.DeepEqual(udevData, device.UDevData) {
+ klog.V(3).Infof("udev data isn't matching for device %s", device.Name)
+ formatStatus.setErr(errUDevDataMismatch, udevDataMismatchReason, formatRetrySuggestion)
+ return
+ }
+ // Check if force is required
+ if v, ok := udevData["ID_FS_TYPE"]; ok {
+ if v != "" && !device.Force {
+ formatStatus.setErr(errForceRequired, fmt.Sprintf("device %s already has a %s fs", device.Name, v), formatRetryWithForceSuggestion)
+ return
+ }
+ }
+ // Format the device
+ fsuuid := uuid.New().String()
+ err = n.makeFS(ctx, device.Path(), fsuuid, device.Force, n.reflinkSupport)
+ if err != nil {
+ klog.Errorf("failed to format drive %s; %w", device.Name, err)
+ formatStatus.setErr(err, "failed to format device", formatRetrySuggestion)
+ return
+ }
+ formatStatus.FSUUID = fsuuid
+ // Mount the device
+ mountTarget := path.Join(consts.MountRootDir, fsuuid)
+ err = n.mountDevice(device.Path(), mountTarget)
+ if err != nil {
+ klog.Errorf("failed to mount drive %s; %w", device.Name, err)
+ formatStatus.setErr(err, "failed to mount device", formatRetrySuggestion)
+ return
+ }
+ formatStatus.mountedAt = mountTarget
+ // probe fsinfo to calculate the allocatedcapacity
+ _, _, totalCapacity, freeCapacity, err = n.probeXFS(device.Path())
+ if err != nil {
+ klog.Errorf("failed to probe XFS for device: %s: %s", device.Name, err.Error())
+ formatStatus.setErr(err, "failed to probe XFS after formatting", formatRetrySuggestion)
+ return
+ }
+ formatStatus.totalCapacity = totalCapacity
+ formatStatus.freeCapacity = freeCapacity
+ // Write metadata
+ if err := writeFormatMetadata(FormatMetadata{
+ FSUUID: fsuuid,
+ FormattedBy: consts.LatestAPIVersion,
+ }, path.Join(mountTarget, metaDataPathSuffix)); err != nil {
+ klog.Errorf("failed to write metadata for device: %s: %s", device.Name, err.Error())
+ formatStatus.setErr(err, "failed to marshal device metadata", formatRetrySuggestion)
+ return
+ }
+ // Create symbolic link
+ if err := os.Symlink(mountTarget, path.Join(mountTarget, fsuuid)); err != nil {
+ klog.Errorf("failed to create symlink for target %s. device: %s err: %s", mountTarget, device.Name, err.Error())
+ formatStatus.setErr(err, "failed to create symlink", formatRetrySuggestion)
+ }
+ return
+}
+
+func (n *nodeAPIHandler) addDrive(ctx context.Context, formatDevice FormatDevice, formatStatus FormatDeviceStatus) error {
+ newDrive := drive.NewDrive(formatStatus.FSUUID, types.DriveStatus{
+ Path: formatDevice.Path(),
+ TotalCapacity: int64(formatStatus.totalCapacity),
+ AllocatedCapacity: int64(formatStatus.totalCapacity - formatStatus.freeCapacity),
+ FreeCapacity: int64(formatStatus.freeCapacity),
+ FSUUID: formatStatus.FSUUID,
+ NodeName: n.nodeID,
+ Status: apiTypes.DriveStatusOK,
+ ModelNumber: formatDevice.Model(),
+ Vendor: formatDevice.Vendor(),
+ AccessTier: apiTypes.AccessTierUnknown,
+ Topology: n.topology,
+ })
+ _, err := client.DriveClient().Create(ctx, newDrive, metav1.CreateOptions{})
+ return err
+}
diff --git a/pkg/rest/ready.go b/pkg/rest/ready.go
new file mode 100644
index 000000000..6c2ee2957
--- /dev/null
+++ b/pkg/rest/ready.go
@@ -0,0 +1,32 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "k8s.io/klog/v2"
+ "net/http"
+)
+
+// readinessHandler - Checks if the process is up. Always returns success.
+func readinessHandler(w http.ResponseWriter, r *http.Request) {
+ klog.V(5).Infof("Received readiness request %v", r)
+ if r.Method != http.MethodGet {
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ } else {
+ w.WriteHeader(http.StatusOK)
+ }
+}
diff --git a/pkg/rest/types.go b/pkg/rest/types.go
new file mode 100644
index 000000000..11d52d2cf
--- /dev/null
+++ b/pkg/rest/types.go
@@ -0,0 +1,97 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+type NodeName string
+
+type Selector string
+
+// DeviceStatusAccessTier denotes device status.
+type DeviceStatus string
+
+const (
+ // DeviceStatusAvailable denotes that the device is available for formatting
+ DeviceStatusAvailable DeviceStatus = "Available"
+
+ // DeviceStatusUnavailable denotes that the device is NOT available for formatting
+ DeviceStatusUnavailable DeviceStatus = "Unavailable"
+)
+
+// GetDevicesRequest is the request type to fetch the devices present in the cluster
+type GetDevicesRequest struct {
+ Nodes Selector `json:"nodes,omitempty"`
+ Drives Selector `json:"drives,omitempty"`
+ Statuses []DeviceStatus `json:"statuses,omitempty"`
+}
+
+// GetDevicesResponse is the response type to represent the devices from the corresponding node
+type GetDevicesResponse struct {
+ DeviceInfo map[NodeName][]Device `json:"deviceInfo"`
+}
+
+// Device holds Disk information
+type Device struct {
+ Name string `json:"name"`
+ MajorMinor string `json:"majorMinor,omitempty"`
+ Size uint64 `json:"size,omitempty"`
+ Model string `json:"model,omitempty"`
+ Vendor string `json:"vendor,omitempty"`
+ Filesystem string `json:"filesystem,omitempty"`
+ Mountpoints []string `json:"mountpoints,omitempty"`
+ Status DeviceStatus `json:"status"`
+ Description string `json:"description"`
+ // UDevData holds the device metadata info probed from `/run/udev/data/b`
+ UDevData map[string]string `json:"udevData,omitempty"`
+}
+
+// FormatDevicesRequest is the request type to represent the format request
+type FormatDevicesRequest struct {
+ FormatInfo map[NodeName][]FormatDevice `json:"formatInfo"`
+}
+
+// FormatDevice represents the devices requested to be formatted
+type FormatDevice struct {
+ Name string `json:"name"`
+ MajorMinor string `json:"majorMinor"`
+ Force bool `json:"force,omitempty"`
+ // UDevData holds the device metadata sent in the fetch drives response
+ UDevData map[string]string `json:"udevData"`
+}
+
+// FormatDevicesResponse represents the format status of the devices requested for formatting
+type FormatDevicesResponse struct {
+ DeviceInfo map[NodeName][]FormatDeviceStatus `json:"deviceInfo"`
+}
+
+// FormatDeviceStatus represents the status of the device requested for formatting
+type FormatDeviceStatus struct {
+ Name string `json:"name"`
+ FSUUID string `json"fsuuid,omitempty`
+ Error string `json:"error,omitempty"`
+ Message string `json:"message,omitempty"`
+ Suggestion string `json:"suggestion,omitempty"`
+ // internals
+ mountedAt string `json:"-"`
+ totalCapacity uint64 `json:"-"`
+ freeCapacity uint64 `json:"-"`
+}
+
+// FormatMetadata represents the format metadata to be saved on the drive
+type FormatMetadata struct {
+ FSUUID string `json:"fsuuid"`
+ FormattedBy string `json:"formattedBy"`
+}
diff --git a/pkg/rest/utils.go b/pkg/rest/utils.go
new file mode 100644
index 000000000..6a6018d43
--- /dev/null
+++ b/pkg/rest/utils.go
@@ -0,0 +1,164 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package rest
+
+import (
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "io"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "os"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/minio/directpv/pkg/device"
+ "github.com/minio/directpv/pkg/types"
+)
+
+type matchFn func(drive *types.Drive, device *device.Device) bool
+
+func getMatchedDevicesForDrive(drive *types.Drive, devices []*device.Device) ([]*device.Device, []*device.Device) {
+ return getMatchedDevices(
+ drive,
+ devices,
+ func(drive *types.Drive, device *device.Device) bool {
+ return fsMatcher(drive, device)
+ },
+ )
+}
+
+func fsMatcher(drive *types.Drive, device *device.Device) bool {
+ if drive.Status.FSUUID != device.FSUUID {
+ return false
+ }
+ return true
+}
+
+func getMatchedDevices(drive *types.Drive, devices []*device.Device, matchFn matchFn) (matchedDevices, unmatchedDevices []*device.Device) {
+ for _, device := range devices {
+ if matchFn(drive, device) {
+ matchedDevices = append(matchedDevices, device)
+ } else {
+ unmatchedDevices = append(unmatchedDevices, device)
+ }
+ }
+ return matchedDevices, unmatchedDevices
+}
+
+func getDeviceNames(devices []*device.Device) string {
+ var deviceNames []string
+ for _, device := range devices {
+ deviceNames = append(deviceNames, device.Name)
+ }
+ return strings.Join(deviceNames, ", ")
+}
+
+// stringIn checks whether value in the slice.
+func stringIn(slice []string, value string) bool {
+ for _, s := range slice {
+ if value == s {
+ return true
+ }
+ }
+ return false
+}
+
+func writeFormatMetadata(formatMetadata FormatMetadata, filePath string) error {
+ if err := os.Mkdir(path.Dir(filePath), 0o777); err != nil && !errors.Is(err, os.ErrExist) {
+ return err
+ }
+ metaDataBytes, err := json.MarshalIndent(formatMetadata, "", "")
+ if err != nil {
+ return err
+ }
+ return ioutil.WriteFile(filePath, metaDataBytes, 0644)
+}
+
+func (d FormatDevice) Path() string {
+ return path.Join("/dev", d.Name)
+}
+
+func (d FormatDevice) Model() string {
+ if d.UDevData == nil {
+ return ""
+ }
+ return d.UDevData["ID_MODEL"]
+}
+
+func (d FormatDevice) Vendor() string {
+ if d.UDevData == nil {
+ return ""
+ }
+ return d.UDevData["ID_VENDOR"]
+}
+
+func (s *FormatDeviceStatus) setErr(err error, message, suggestion string) {
+ s.Error = err.Error()
+ s.Message = message
+ s.Suggestion = suggestion
+}
+
+func getTransport() *http.Transport {
+ // Keep TLS config.
+ tlsConfig := &tls.Config{
+ // Can't use SSLv3 because of POODLE and BEAST
+ // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+ // Can't use TLSv1.1 because of RC4 cipher usage
+ MinVersion: tls.VersionTLS12,
+ InsecureSkipVerify: true, // FIXME: use trusted CA
+ }
+ return &http.Transport{
+ Proxy: http.ProxyFromEnvironment,
+ DialContext: (&net.Dialer{
+ Timeout: 30 * time.Second,
+ }).DialContext,
+ MaxIdleConnsPerHost: 1024,
+ IdleConnTimeout: 30 * time.Second,
+ ResponseHeaderTimeout: 1 * time.Minute,
+ TLSHandshakeTimeout: 5 * time.Second,
+ ExpectContinueTimeout: 5 * time.Second,
+ TLSClientConfig: tlsConfig,
+ // Go net/http automatically unzip if content-type is
+ // gzip disable this feature, as we are always interested
+ // in raw stream.
+ DisableCompression: true,
+ }
+}
+
+// drainBody close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func drainBody(respBody io.ReadCloser) {
+ // Callers should close resp.Body when done reading from it.
+ // If resp.Body is not closed, the Client's underlying RoundTripper
+ // (typically Transport) may not be able to re-use a persistent TCP
+ // connection to the server for a subsequent "keep-alive" request.
+ if respBody != nil {
+ // Drain any remaining Body and then close the connection.
+ // Without this closing connection would disallow re-using
+ // the same connection for future uses.
+ // - http://stackoverflow.com/a/17961593/4465767
+ defer respBody.Close()
+ io.Copy(ioutil.Discard, respBody)
+ }
+}
diff --git a/pkg/sys/mount_linux.go b/pkg/sys/mount_linux.go
index 63cc3a690..6fcbc574c 100644
--- a/pkg/sys/mount_linux.go
+++ b/pkg/sys/mount_linux.go
@@ -145,7 +145,7 @@ func mount(proc1Mountinfo, device, target, fsType string, flags []string, superB
}
mountFlags |= value
}
-
+ klog.V(3).InfoS("mouting device", "device", device, "target", target, "fsType", fsType, "mountFlags", mountFlags, "superBlockFlags", superBlockFlags)
return syscall.Mount(device, target, fsType, mountFlags, superBlockFlags)
}
@@ -171,7 +171,7 @@ func bindMount(proc1Mountinfo, source, target, fsType string, recursive, readOnl
if readOnly {
flags |= mountFlagMap["ro"]
}
- klog.V(5).InfoS("bind mounting directory", "source", source, "target", target, "fsType", fsType, "recursive", recursive, "readOnly", readOnly, "superBlockFlags", superBlockFlags)
+ klog.V(3).InfoS("bind mounting directory", "source", source, "target", target, "fsType", fsType, "recursive", recursive, "readOnly", readOnly, "superBlockFlags", superBlockFlags)
return syscall.Mount(source, target, fsType, flags, superBlockFlags)
}
@@ -182,7 +182,7 @@ func unmount(proc1Mountinfo, target string, force, detach, expire bool) error {
}
if _, found := mountPointMap[target]; !found {
- klog.V(5).InfoS("target already unmounted", "target", target, "force", force, "detach", detach, "expire", expire)
+ klog.V(3).InfoS("target already unmounted", "target", target, "force", force, "detach", detach, "expire", expire)
return nil
}
@@ -197,6 +197,6 @@ func unmount(proc1Mountinfo, target string, force, detach, expire bool) error {
flags |= syscall.MNT_EXPIRE
}
- klog.V(5).InfoS("unmounting mount point", "target", target, "force", force, "detach", detach, "expire", expire)
+ klog.V(3).InfoS("unmounting mount point", "target", target, "force", force, "detach", detach, "expire", expire)
return syscall.Unmount(target, flags)
}
diff --git a/pkg/types/aliases.go b/pkg/types/aliases.go
index 685b5656f..5b9436825 100644
--- a/pkg/types/aliases.go
+++ b/pkg/types/aliases.go
@@ -29,6 +29,7 @@ var Versions = []string{
}
var LatestAddToScheme = directpv.AddToScheme
+
type Interface = typeddirectpv.DirectpvV1beta1Interface
type Client = typeddirectpv.DirectpvV1beta1Client
diff --git a/pkg/xfs/mount.go b/pkg/xfs/mount.go
index 81e9a108d..47b2c5bb0 100644
--- a/pkg/xfs/mount.go
+++ b/pkg/xfs/mount.go
@@ -1,5 +1,3 @@
-//go:build linux
-
// This file is part of MinIO DirectPV
// Copyright (c) 2022 MinIO, Inc.
//
diff --git a/pkg/xfs/probe.go b/pkg/xfs/probe.go
new file mode 100644
index 000000000..e531ab972
--- /dev/null
+++ b/pkg/xfs/probe.go
@@ -0,0 +1,21 @@
+// This file is part of MinIO DirectPV
+// Copyright (c) 2021, 2022 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package xfs
+
+func Probe(path string) (fsuuid, label string, totalCapacity, freeCapacity uint64, err error) {
+ return probe(path)
+}
diff --git a/pkg/xfs/xfs.go b/pkg/xfs/probe_linux.go
similarity index 74%
rename from pkg/xfs/xfs.go
rename to pkg/xfs/probe_linux.go
index ebebc4d9b..88bdf191f 100644
--- a/pkg/xfs/xfs.go
+++ b/pkg/xfs/probe_linux.go
@@ -1,3 +1,5 @@
+//go:build linux
+
// This file is part of MinIO DirectPV
// Copyright (c) 2021, 2022 MinIO, Inc.
//
@@ -18,10 +20,13 @@ package xfs
import (
"bytes"
+ "context"
"encoding/binary"
"errors"
"fmt"
"io"
+ "os"
+ "time"
)
// MinSupportedDeviceSize is minimum supported size for default XFS filesystem.
@@ -77,8 +82,7 @@ type superBlock struct {
// Ignoring the rest
}
-// Probe probes FSUUID, total and free capacity.
-func Probe(reader io.Reader) (FSUUID, label string, totalCapacity, freeCapacity uint64, err error) {
+func readSuperBlock(reader io.Reader) (FSUUID, label string, totalCapacity, freeCapacity uint64, err error) {
var sb superBlock
if err = binary.Read(reader, binary.BigEndian, &sb); err != nil {
return
@@ -95,3 +99,31 @@ func Probe(reader io.Reader) (FSUUID, label string, totalCapacity, freeCapacity
return
}
+
+// probe probes FSUUID, total and free capacity.
+func probe(path string) (fsuuid, label string, totalCapacity, freeCapacity uint64, err error) {
+ ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancelFunc()
+
+ doneCh := make(chan struct{})
+ go func() {
+ var devFile *os.File
+ devFile, err = os.OpenFile(path, os.O_RDONLY, os.ModeDevice)
+ if err != nil {
+ return
+ }
+ defer devFile.Close()
+ // only XFS is the supported filesystem as of now
+ fsuuid, label, totalCapacity, freeCapacity, err = readSuperBlock(devFile)
+ close(doneCh)
+ }()
+
+ select {
+ case <-ctx.Done():
+ err = fmt.Errorf("%w; %v", ErrCanceled, ctx.Err())
+ return
+ case <-doneCh:
+ }
+
+ return fsuuid, label, totalCapacity, freeCapacity, err
+}
diff --git a/pkg/sys/sys_linux.go b/pkg/xfs/probe_other.go
similarity index 75%
rename from pkg/sys/sys_linux.go
rename to pkg/xfs/probe_other.go
index 1e755095a..8166f4995 100644
--- a/pkg/sys/sys_linux.go
+++ b/pkg/xfs/probe_other.go
@@ -1,4 +1,4 @@
-//go:build linux
+//go:build !linux
// This file is part of MinIO DirectPV
// Copyright (c) 2021, 2022 MinIO, Inc.
@@ -16,13 +16,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package sys
+package xfs
-import "path/filepath"
+import (
+ "fmt"
+ "runtime"
+)
-func getDeviceByFSUUID(fsuuid string) (device string, err error) {
- if device, err = filepath.EvalSymlinks("/dev/disk/by-uuid/" + fsuuid); err == nil {
- device = filepath.ToSlash(device)
- }
+func probe(path string) (fsuuid, label string, totalCapacity, freeCapacity uint64, err error) {
+ err = fmt.Errorf("unsupported operating system %v", runtime.GOOS)
return
}