forked from canonical/lxd
/
endpoints.go
409 lines (351 loc) · 11 KB
/
endpoints.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
package endpoints
import (
"fmt"
"net"
"net/http"
"sync"
"time"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
tomb "gopkg.in/tomb.v2"
)
// Config holds various configuration values that affect LXD endpoints
// initialization.
type Config struct {
// The LXD var directory to create Unix sockets in.
Dir string
// UnixSocket is the path to the Unix socket to bind
UnixSocket string
// HTTP server handling requests for the LXD RESTful API.
RestServer *http.Server
// HTTP server for the internal /dev/lxd API exposed to containers.
DevLxdServer *http.Server
// The TLS keypair and optional CA to use for the network endpoint. It
// must be always provided, since the pubblic key will be included in
// the response of the /1.0 REST API as part of the server info.
//
// It can be updated after the endpoints are up using NetworkUpdateCert().
Cert *shared.CertInfo
// System group name to which the unix socket for the local endpoint should be
// chgrp'ed when starting. The default is to use the process group. An empty
// string means "use the default".
LocalUnixSocketGroup string
// NetworkSetAddress sets the address for the network endpoint. If not
// set, the network endpoint won't be started (unless it's passed via
// socket-based activation).
//
// It can be updated after the endpoints are up using NetworkUpdateAddress().
NetworkAddress string
// Optional dedicated network address for clustering traffic. If not
// set, NetworkAddress will be used.
//
// It can be updated after the endpoints are up using ClusterUpdateAddress().
ClusterAddress string
// DebugSetAddress sets the address for the pprof endpoint.
//
// It can be updated after the endpoints are up using PprofUpdateAddress().
DebugAddress string
}
// Up brings up all applicable LXD endpoints and starts accepting HTTP
// requests.
//
// The endpoints will be activated in the following order and according to the
// following rules:
//
// local endpoint (unix socket)
// ----------------------------
//
// If socket-based activation is detected, look for a unix socket among the
// inherited file descriptors and use it for the local endpoint (or if no such
// file descriptor exists, don't bring up the local endpoint at all).
//
// If no socket-based activation is detected, create a unix socket using the
// default <lxd-var-dir>/unix.socket path. The file mode of this socket will be set
// to 660, the file owner will be set to the process' UID, and the file group
// will be set to the process GID, or to the GID of the system group name
// specified via config.LocalUnixSocketGroup.
//
// devlxd endpoint (unix socket)
// ----------------------------
//
// Created using <lxd-var-dir>/devlxd/sock, with file mode set to 666 (actual
// authorization will be performed by the HTTP server using the socket ucred
// struct).
//
// remote endpoint (TCP socket with TLS)
// -------------------------------------
//
// If socket-based activation is detected, look for a network socket among the
// inherited file descriptors and use it for the network endpoint.
//
// If a network address was set via config.NetworkAddress, then close any listener
// that was detected via socket-based activation and create a new network
// socket bound to the given address.
//
// The network endpoint socket will use TLS encryption, using the certificate
// keypair and CA passed via config.Cert.
//
// cluster endpoint (TCP socket with TLS)
// -------------------------------------
//
// If a network address was set via config.ClusterAddress, then attach
// config.RestServer to it.
func Up(config *Config) (*Endpoints, error) {
if config.Dir == "" {
return nil, fmt.Errorf("No directory configured")
}
if config.UnixSocket == "" {
return nil, fmt.Errorf("No unix socket configured")
}
if config.RestServer == nil {
return nil, fmt.Errorf("No REST server configured")
}
if config.DevLxdServer == nil {
return nil, fmt.Errorf("No devlxd server configured")
}
if config.Cert == nil {
return nil, fmt.Errorf("No TLS certificate configured")
}
endpoints := &Endpoints{
systemdListenFDsStart: util.SystemdListenFDsStart,
}
err := endpoints.up(config)
if err != nil {
endpoints.Down()
return nil, err
}
return endpoints, nil
}
// Endpoints are in charge of bringing up and down the HTTP endpoints for
// serving the LXD RESTful API.
//
// When LXD starts up, they start listen to the appropriate sockets and attach
// the relevant HTTP handlers to them. When LXD shuts down they close all
// sockets.
type Endpoints struct {
tomb *tomb.Tomb // Controls the HTTP servers shutdown.
mu sync.RWMutex // Serialize access to internal state.
listeners map[kind]net.Listener // Activer listeners by endpoint type.
servers map[kind]*http.Server // HTTP servers by endpoint type.
cert *shared.CertInfo // Keypair and CA to use for TLS.
inherited map[kind]bool // Store whether the listener came through socket activation
systemdListenFDsStart int // First socket activation FD, for tests.
}
// Up brings up all configured endpoints and starts accepting HTTP requests.
func (e *Endpoints) up(config *Config) error {
e.mu.Lock()
defer e.mu.Unlock()
e.servers = map[kind]*http.Server{
devlxd: config.DevLxdServer,
local: config.RestServer,
network: config.RestServer,
cluster: config.RestServer,
pprof: pprofCreateServer(),
}
e.cert = config.Cert
e.inherited = map[kind]bool{}
var err error
// Check for socket activation.
systemdListeners := util.GetListeners(e.systemdListenFDsStart)
if len(systemdListeners) > 0 {
e.listeners = activatedListeners(systemdListeners, e.cert)
for kind := range e.listeners {
e.inherited[kind] = true
}
} else {
e.listeners = map[kind]net.Listener{}
e.listeners[local], err = localCreateListener(config.UnixSocket, config.LocalUnixSocketGroup)
if err != nil {
return fmt.Errorf("local endpoint: %v", err)
}
}
// Start the devlxd listener
e.listeners[devlxd], err = createDevLxdlListener(config.Dir)
if err != nil {
return err
}
if config.NetworkAddress != "" {
listener, ok := e.listeners[network]
if ok {
logger.Infof("Replacing inherited TCP socket with configured one")
listener.Close()
e.inherited[network] = false
}
// Errors here are not fatal and are just logged (unless we're clustered, see below).
var networkAddressErr error
attempts := 0
againHttps:
e.listeners[network], networkAddressErr = networkCreateListener(config.NetworkAddress, e.cert)
isCovered := util.IsAddressCovered(config.ClusterAddress, config.NetworkAddress)
if config.ClusterAddress != "" {
if isCovered {
// In case of clustering we fail if we can't bind the network address.
if networkAddressErr != nil {
if attempts == 0 {
logger.Infof("Unable to bind https address %q, re-trying for a minute", config.NetworkAddress)
}
attempts++
if attempts < 60 {
time.Sleep(1 * time.Second)
goto againHttps
}
return networkAddressErr
}
} else {
againCluster:
e.listeners[cluster], err = networkCreateListener(config.ClusterAddress, e.cert)
if err != nil {
if attempts == 0 {
logger.Infof("Unable to bind cluster address %q, re-trying for a minute", config.ClusterAddress)
}
attempts++
if attempts < 60 {
time.Sleep(1 * time.Second)
goto againCluster
}
return err
}
}
logger.Infof("Starting cluster handler:")
e.serveHTTP(cluster)
} else if networkAddressErr != nil {
logger.Error("Cannot currently listen on https socket, re-trying once in 30s...", log.Ctx{"err": networkAddressErr})
go func() {
time.Sleep(30 * time.Second)
err := e.NetworkUpdateAddress(config.NetworkAddress)
if err != nil {
logger.Error("Still unable to listen on https socket", log.Ctx{"err": err})
}
}()
}
}
if config.DebugAddress != "" {
e.listeners[pprof], err = pprofCreateListener(config.DebugAddress)
if err != nil {
return err
}
logger.Infof("Starting pprof handler:")
e.serveHTTP(pprof)
}
logger.Infof("Starting /dev/lxd handler:")
e.serveHTTP(devlxd)
logger.Infof("REST API daemon:")
e.serveHTTP(local)
e.serveHTTP(network)
return nil
}
// Down brings down all endpoints and stops serving HTTP requests.
func (e *Endpoints) Down() error {
e.mu.Lock()
defer e.mu.Unlock()
if e.listeners[network] != nil || e.listeners[local] != nil {
logger.Infof("Stopping REST API handler:")
err := e.closeListener(network)
if err != nil {
return err
}
err = e.closeListener(local)
if err != nil {
return err
}
}
if e.listeners[cluster] != nil {
logger.Infof("Stopping cluster handler:")
err := e.closeListener(cluster)
if err != nil {
return err
}
}
if e.listeners[devlxd] != nil {
logger.Infof("Stopping /dev/lxd handler:")
err := e.closeListener(devlxd)
if err != nil {
return err
}
}
if e.listeners[pprof] != nil {
logger.Infof("Stopping pprof handler:")
err := e.closeListener(pprof)
if err != nil {
return err
}
}
if e.tomb != nil {
e.tomb.Kill(nil)
e.tomb.Wait()
}
return nil
}
// Start an HTTP server for the endpoint associated with the given code.
func (e *Endpoints) serveHTTP(kind kind) {
listener := e.listeners[kind]
if listener == nil {
return
}
ctx := log.Ctx{"socket": listener.Addr()}
if e.inherited[kind] {
ctx["inherited"] = true
}
message := fmt.Sprintf(" - binding %s", descriptions[kind])
logger.Info(message, ctx)
server := e.servers[kind]
// Defer the creation of the tomb, so Down() doesn't wait on it unless
// we actually have spawned at least a server.
if e.tomb == nil {
e.tomb = &tomb.Tomb{}
}
e.tomb.Go(func() error {
server.Serve(listener)
return nil
})
}
// Stop the HTTP server of the endpoint associated with the given code. The
// associated socket will be shutdown too.
func (e *Endpoints) closeListener(kind kind) error {
listener := e.listeners[kind]
if listener == nil {
return nil
}
delete(e.listeners, kind)
logger.Info(" - closing socket", log.Ctx{"socket": listener.Addr()})
return listener.Close()
}
// Use the listeners associated with the file descriptors passed via
// socket-based activation.
func activatedListeners(systemdListeners []net.Listener, cert *shared.CertInfo) map[kind]net.Listener {
listeners := map[kind]net.Listener{}
for _, listener := range systemdListeners {
var kind kind
switch listener.(type) {
case *net.UnixListener:
kind = local
case *net.TCPListener:
kind = network
listener = networkTLSListener(listener, cert)
default:
continue
}
listeners[kind] = listener
}
return listeners
}
// Numeric code identifying a specific API endpoint type.
type kind int
// Numeric codes identifying the various endpoints.
const (
local kind = iota
devlxd
network
pprof
cluster
)
// Human-readable descriptions of the various kinds of endpoints.
var descriptions = map[kind]string{
local: "Unix socket",
devlxd: "devlxd socket",
network: "TCP socket",
pprof: "pprof socket",
cluster: "cluster socket",
}