/
run.go
212 lines (187 loc) · 6.43 KB
/
run.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
package app
import (
"context"
"errors"
"fmt"
"net"
"os"
"os/signal"
"syscall"
"time"
"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/spf13/viper"
"github.com/topolvm/topolvm"
topolvmlegacyv1 "github.com/topolvm/topolvm/api/legacy/v1"
topolvmv1 "github.com/topolvm/topolvm/api/v1"
clientwrapper "github.com/topolvm/topolvm/internal/client"
"github.com/topolvm/topolvm/internal/controller"
"github.com/topolvm/topolvm/internal/driver"
"github.com/topolvm/topolvm/internal/lvmd"
"github.com/topolvm/topolvm/internal/lvmd/command"
"github.com/topolvm/topolvm/internal/runners"
"github.com/topolvm/topolvm/pkg/lvmd/proto"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/health/grpc_health_v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/yaml"
//+kubebuilder:scaffold:imports
)
var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
cfgFilePath string
)
func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))
utilruntime.Must(topolvmv1.AddToScheme(scheme))
utilruntime.Must(topolvmlegacyv1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
}
func subMain(ctx context.Context) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
nodename := viper.GetString("nodename")
if len(nodename) == 0 {
return errors.New("node name is not given")
}
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&config.zapOpts)))
metricsServerOptions := metricsserver.Options{
BindAddress: config.metricsAddr,
}
if config.secureMetricsServer {
metricsServerOptions.SecureServing = true
metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsServerOptions,
LeaderElection: false,
})
if err != nil {
setupLog.Error(err, "unable to start manager")
return err
}
client := clientwrapper.NewWrappedClient(mgr.GetClient())
apiReader := clientwrapper.NewWrappedReader(mgr.GetAPIReader(), mgr.GetClient().Scheme())
var lvService proto.LVServiceClient
var vgService proto.VGServiceClient
var health grpc_health_v1.HealthClient
if config.embedLvmd {
command.Containerized = true
if err := loadConfFile(ctx, cfgFilePath); err != nil {
return err
}
dcm := lvmd.NewDeviceClassManager(config.lvmd.DeviceClasses)
ocm := lvmd.NewLvcreateOptionClassManager(config.lvmd.LvcreateOptionClasses)
lvService, vgService = lvmd.NewEmbeddedServiceClients(ctx, dcm, ocm)
} else {
dialer := &net.Dialer{}
dialFunc := func(ctx context.Context, a string) (net.Conn, error) {
return dialer.DialContext(ctx, "unix", a)
}
conn, err := grpc.Dial(config.lvmdSocket, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialFunc))
if err != nil {
return err
}
defer conn.Close()
lvService, vgService = proto.NewLVServiceClient(conn), proto.NewVGServiceClient(conn)
health = grpc_health_v1.NewHealthClient(conn)
}
lvcontroller := controller.NewLogicalVolumeReconcilerWithServices(client, nodename, vgService, lvService)
if err := lvcontroller.SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "LogicalVolume")
return err
}
//+kubebuilder:scaffold:builder
// Add health checker to manager
checker := runners.NewChecker(checkFunc(health, apiReader), 1*time.Minute) // adjusted signature
if err := mgr.Add(checker); err != nil {
return err
}
// Add metrics exporter to manager.
// Note that grpc.ClientConn can be shared with multiple stubs/services.
// https://github.com/grpc/grpc-go/tree/master/examples/features/multiplex
if err := mgr.Add(runners.NewMetricsExporter(vgService, client, nodename)); err != nil { // adjusted signature
return err
}
// Add gRPC server to manager.
if err := os.MkdirAll(topolvm.DeviceDirectory, 0755); err != nil {
return err
}
grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ErrorLoggingInterceptor))
csi.RegisterIdentityServer(grpcServer, driver.NewIdentityServer(checker.Ready))
nodeServer, err := driver.NewNodeServer(nodename, vgService, lvService, mgr) // adjusted signature
if err != nil {
return err
}
csi.RegisterNodeServer(grpcServer, nodeServer)
err = mgr.Add(runners.NewGRPCRunner(grpcServer, config.csiSocket, false))
if err != nil {
return err
}
c := make(chan os.Signal, 2)
signal.Notify(c, []os.Signal{os.Interrupt, syscall.SIGTERM}...)
go func() {
<-c
cancel()
<-c
os.Exit(1) // second signal. Exit directly.
}()
setupLog.Info("starting manager")
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
return err
}
return nil
}
//+kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers,verbs=get;list;watch
func checkFunc(health grpc_health_v1.HealthClient, r client.Reader) func() error {
return func() error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if health != nil {
res, err := health.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
if err != nil {
return err
}
if status := res.GetStatus(); status != grpc_health_v1.HealthCheckResponse_SERVING {
return fmt.Errorf("lvmd does not working: %s", status.String())
}
}
var drv storagev1.CSIDriver
return r.Get(ctx, types.NamespacedName{Name: topolvm.GetPluginName()}, &drv)
}
}
func ErrorLoggingInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
resp, err = handler(ctx, req)
if err != nil {
ctrl.Log.Error(err, "error on grpc call", "method", info.FullMethod)
}
return resp, err
}
func loadConfFile(ctx context.Context, cfgFilePath string) error {
b, err := os.ReadFile(cfgFilePath)
if err != nil {
return err
}
err = yaml.Unmarshal(b, &config.lvmd)
if err != nil {
return err
}
log.FromContext(ctx).Info("configuration file loaded",
"device_classes", config.lvmd.DeviceClasses,
"file_name", cfgFilePath,
)
return nil
}