-
Notifications
You must be signed in to change notification settings - Fork 230
/
spec.go
453 lines (373 loc) · 13.6 KB
/
spec.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
package dockerbuild
import (
"fmt"
pathpkg "path"
"path/filepath"
"slices"
strconv "strconv"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/rs/xid"
"encr.dev/pkg/builder"
"encr.dev/pkg/noopgateway"
"encr.dev/pkg/noopgwdesc"
"encr.dev/pkg/option"
"encr.dev/pkg/paths"
"encr.dev/pkg/supervisor"
meta "encr.dev/proto/encore/parser/meta/v1"
)
type ImageSpecFile struct {
Images []*ImageSpec
}
// ImageSpec is a specification for how to build a docker image.
type ImageSpec struct {
// The entrypoint to use for the image. It must be non-empty.
// The first entry is the executable path, and the rest are the arguments.
Entrypoint []string
// Environment variables to set for the entrypoint.
Env []string
// The working dir to use for executing the entrypoint.
WorkingDir ImagePath
// BuildInfo contains information about the build.
BuildInfo BuildInfoSpec
// A map from the builder filesystem paths to the destination path in the image.
// If the source is a directory, it will be copied recursively.
CopyData map[ImagePath]HostPath
// Whether to bundle source into the image.
// It's handled separately from CopyData since we apply some filtering
// on what's copied, like excluding .git directories and other build artifacts.
BundleSource option.Option[BundleSourceSpec]
// Supervisor specifies the supervisor configuration.
Supervisor option.Option[SupervisorSpec]
// The names of services bundled in this image.
BundledServices []string
// The names of gateways bundled in this image.
BundledGateways []string
// The docker base image to use. If None it defaults to the empty scratch image.
DockerBaseImage string
// StargzPrioritizedFiles are file paths in the image that should be prioritized for
// stargz compression, allowing for faster streaming of those files.
StargzPrioritizedFiles []ImagePath
// FeatureFlags specifies feature flags enabled for this image.
FeatureFlags map[FeatureFlag]bool
// The app metadata, protobuf encoded.
Meta []byte
}
type BuildInfoSpec struct {
// The build info to include in the image.
Info BuildInfo
// The path in the image where the build info is written, as a JSON file.
InfoPath ImagePath
}
type BuildInfo struct {
// The version of Encore with which the app was compiled.
// This is string is for informational use only, and its format should not be relied on.
EncoreCompiler string
// AppCommit describes the commit of the app.
AppCommit CommitInfo
}
type CommitInfo struct {
Revision string
Uncommitted bool
}
type BundleSourceSpec struct {
Source HostPath
Dest ImagePath
// Source paths to exclude from copying, relative to Source.
ExcludeSource []RelPath
}
type SupervisorSpec struct {
// Where to mount the supervisor binary in the image.
MountPath ImagePath
// Where to write the supervisor configuration in the image.
ConfigPath ImagePath
// The config to pass to the supervisor.
Config *supervisor.Config
}
type DescribeConfig struct {
// The parsed metadata.
Meta *meta.Data
// The compile result.
Compile *builder.CompileResult
// The directory containing the runtimes.
Runtimes HostPath
// The docker base image to use, if any. If None it defaults to the empty scratch image.
DockerBaseImage option.Option[string]
// BundleSource specifies whether to bundle source into the image,
// and where the source is located on the host filesystem.
BundleSource option.Option[BundleSourceSpec]
// WorkingDir specifies the working directory to start the docker image in.
WorkingDir option.Option[ImagePath]
// BuildInfo contains information about the build.
BuildInfo BuildInfo
}
type (
// HostPath is a path on the host filesystem.
HostPath string
// ImagePath is a path in the docker image.
ImagePath string
// RelPath is a relative path.
RelPath string
)
func (i ImagePath) Dir() ImagePath { return ImagePath(pathpkg.Dir(string(i))) }
func (i ImagePath) Clean() ImagePath { return ImagePath(pathpkg.Clean(string(i))) }
func (i ImagePath) String() string { return string(i) }
func (i ImagePath) Join(p ...string) ImagePath {
return ImagePath(pathpkg.Join(string(i), pathpkg.Join(p...)))
}
func (i ImagePath) JoinImage(p ImagePath) ImagePath {
return i.Join(string(p))
}
func (h HostPath) Dir() HostPath { return HostPath(filepath.Dir(string(h))) }
func (h HostPath) Join(p ...string) HostPath {
return HostPath(filepath.Join(string(h), filepath.Join(p...)))
}
func (h HostPath) JoinHost(p HostPath) HostPath {
return h.Join(string(p))
}
func (h HostPath) ToImage() ImagePath {
return ImagePath(filepath.ToSlash(string(h)))
}
func (h HostPath) String() string { return string(h) }
func (h HostPath) Rel(target HostPath) (HostPath, error) {
rel, err := filepath.Rel(string(h), string(target))
return HostPath(rel), err
}
func (h HostPath) IsAbs() bool {
return filepath.IsAbs(h.String())
}
// Describe describes the docker image to build.
func Describe(cfg DescribeConfig) (*ImageSpec, error) {
return newImageSpecBuilder().Describe(cfg)
}
func newImageSpecBuilder() *imageSpecBuilder {
return &imageSpecBuilder{
procIDGen: randomProcID,
spec: &ImageSpec{
CopyData: make(map[ImagePath]HostPath),
FeatureFlags: make(map[FeatureFlag]bool),
BundledGateways: []string{},
BundledServices: []string{},
},
seenArtifactDirs: make(map[HostPath]*imageArtifactDir),
seenPrioFiles: make(map[ImagePath]bool),
}
}
type imageArtifactDir struct {
Base ImagePath
BuildArtifacts ImagePath
}
type imageSpecBuilder struct {
spec *ImageSpec
// procIDGen generates a random id for each process.
// Defaults to randomProcID.
procIDGen func() string
// The artifact dirs we've already seen, to avoid
// duplicate copies into the image.
seenArtifactDirs map[HostPath]*imageArtifactDir
seenPrioFiles map[ImagePath]bool
}
const (
// defaultSupervisorMountPath is the path in the image where the supervisor is mounted.
defaultSupervisorMountPath ImagePath = "/encore/bin/supervisor"
// defaultSupervisorConfigPath is the path in the image where the supervisor config is located.
defaultSupervisorConfigPath ImagePath = "/encore/supervisor.config.json"
// defaultBuildInfoPath is the path in the image where the build information is located.
defaultBuildInfoPath ImagePath = "/encore/build-info.json"
// defaultMetaPath is the path in the image where the application metadata is located.
defaultMetaPath ImagePath = "/encore/meta"
)
func (b *imageSpecBuilder) Describe(cfg DescribeConfig) (*ImageSpec, error) {
// Allocate artifact directories for each output.
for _, out := range cfg.Compile.Outputs {
b.allocArtifactDir(out)
}
// Determine if we should use the supervisor.
// We should use it in all cases, except where we have a single Go entrypoint.
useSupervisor := true
if len(cfg.Compile.Outputs) == 1 && len(cfg.Compile.Outputs[0].GetEntrypoints()) == 1 {
ep := cfg.Compile.Outputs[0].GetEntrypoints()[0]
if out, ok := cfg.Compile.Outputs[0].(*builder.GoBuildOutput); ok {
imageArtifacts, ok := b.seenArtifactDirs[HostPath(out.GetArtifactDir())]
if !ok {
return nil, errors.Errorf("missing image artifact dir for %q", out.GetArtifactDir())
}
cmd := ep.Cmd.Expand(paths.FS(imageArtifacts.BuildArtifacts))
b.spec.Entrypoint = cmd.Command
b.spec.Env = cmd.Env
useSupervisor = false
} else if out, ok := cfg.Compile.Outputs[0].(*builder.JSBuildOutput); ok {
imageArtifacts, ok := b.seenArtifactDirs[HostPath(out.GetArtifactDir())]
if !ok {
return nil, errors.Errorf("missing image artifact dir for %q", out.GetArtifactDir())
}
cmd := ep.Cmd.Expand(paths.FS(imageArtifacts.BuildArtifacts))
b.spec.Entrypoint = cmd.Command
b.spec.Env = cmd.Env
useSupervisor = false
// If we have a supervisor, we need to use the new runtime config.
b.spec.FeatureFlags[NewRuntimeConfig] = true
}
}
if useSupervisor {
config := &supervisor.Config{
NoopGateways: make(map[string]*noopgateway.Description),
}
super := SupervisorSpec{
MountPath: defaultSupervisorMountPath,
ConfigPath: defaultSupervisorConfigPath,
Config: config,
}
seenGateways := make(map[string]bool)
for _, out := range cfg.Compile.Outputs {
imageArtifacts, ok := b.seenArtifactDirs[HostPath(out.GetArtifactDir())]
if !ok {
return nil, errors.Errorf("missing image artifact dir for %q", out.GetArtifactDir())
}
for _, ep := range out.GetEntrypoints() {
cmd := ep.Cmd.Expand(paths.FS(imageArtifacts.BuildArtifacts))
proc := supervisor.Proc{
ID: b.procIDGen(),
Command: cmd.Command,
Env: cmd.Env,
Services: slices.Clone(ep.Services),
Gateways: slices.Clone(ep.Gateways),
}
slices.Sort(proc.Services)
slices.Sort(proc.Gateways)
for _, gw := range ep.Gateways {
seenGateways[gw] = true
}
config.Procs = append(config.Procs, proc)
}
}
// We need all gateways to be provided by some docker image. But for now, since we only support
// a single docker image, we need all gateways to be provided by this image.
// Each gateway that's not hosted by this image should be provided by a noop-gateway.
if cfg.Meta != nil { // nil check for backwards compatibility
for _, gw := range cfg.Meta.Gateways {
if !seenGateways[gw.EncoreName] {
config.NoopGateways[gw.EncoreName] = noopgwdesc.Describe(cfg.Meta, nil)
}
}
}
b.addPrio(super.MountPath)
b.spec.Supervisor = option.Some(super)
b.spec.Entrypoint = []string{string(super.MountPath), "-c", string(super.ConfigPath)}
b.spec.Env = nil // not needed by supervisor
// If we have a supervisor, we need to use the new runtime config.
b.spec.FeatureFlags[NewRuntimeConfig] = true
}
// Compute bundled services and gateways.
{
for _, out := range cfg.Compile.Outputs {
for _, ep := range out.GetEntrypoints() {
b.spec.BundledServices = append(b.spec.BundledServices, ep.Services...)
b.spec.BundledGateways = append(b.spec.BundledGateways, ep.Gateways...)
}
}
// If we have any noop-gateways, consider them bundled, too.
if super, ok := b.spec.Supervisor.Get(); ok {
for name := range super.Config.NoopGateways {
b.spec.BundledGateways = append(b.spec.BundledGateways, name)
}
}
// Sort and deduplicate.
slices.Sort(b.spec.BundledServices)
slices.Compact(b.spec.BundledServices)
slices.Sort(b.spec.BundledGateways)
slices.Compact(b.spec.BundledGateways)
}
// Add entrypoint files to prioritized files.
for _, out := range cfg.Compile.Outputs {
hostArtifacts := HostPath(out.GetArtifactDir())
imageArtifacts, ok := b.seenArtifactDirs[hostArtifacts]
if !ok {
return nil, errors.Errorf("missing image artifact dir for %q", hostArtifacts)
}
// If this is a JS build, copy the node modules and package.json to out dir.
if jsOut, ok := out.(*builder.JSBuildOutput); ok {
if nodeModules, ok := jsOut.NodeModules.Get(); ok {
dst := imageArtifacts.Base.Join("node_modules")
b.spec.CopyData[dst] = HostPath(nodeModules)
}
pkgJsonPath := imageArtifacts.Base.Join("package.json")
b.spec.CopyData[pkgJsonPath] = HostPath(jsOut.PackageJson)
b.addPrio(pkgJsonPath)
}
for _, ep := range out.GetEntrypoints() {
// For each entrypoint, add prioritized files.
files := ep.Cmd.PrioritizedFiles.Expand(paths.FS(imageArtifacts.BuildArtifacts))
for _, file := range files {
b.addPrio(ImagePath(file))
}
}
}
// If we have any JS outputs that need the local runtime, copy it into the image.
{
for _, out := range cfg.Compile.Outputs {
if _, ok := out.(*builder.JSBuildOutput); ok {
// Include the encore.dev package, at the same location.
runtimeSrc := cfg.Runtimes.Join("js", "encore.dev")
b.spec.CopyData[runtimeSrc.ToImage()] = runtimeSrc
// Add the encore-runtime.node file, and set the environment variable to point to it.
nativeRuntimeHost := cfg.Runtimes.Join("js", "encore-runtime.node")
nativeRuntimeImg := nativeRuntimeHost.ToImage()
b.spec.CopyData[nativeRuntimeImg] = nativeRuntimeHost
b.spec.Env = append(b.spec.Env, fmt.Sprintf("ENCORE_RUNTIME_LIB=%s", nativeRuntimeImg))
b.addPrio(nativeRuntimeImg)
break
}
}
}
b.spec.DockerBaseImage = cfg.DockerBaseImage.GetOrElse("scratch")
b.spec.BundleSource = cfg.BundleSource
b.spec.WorkingDir = cfg.WorkingDir.GetOrElse("/")
// Include build information.
b.spec.BuildInfo = BuildInfoSpec{
Info: cfg.BuildInfo,
InfoPath: defaultBuildInfoPath,
}
// Include the app metadata.
{
md, err := proto.Marshal(cfg.Meta)
if err != nil {
return nil, errors.Wrap(err, "marshal meta")
}
b.spec.Meta = md
}
return b.spec, nil
}
func (b *imageSpecBuilder) addPrio(path ImagePath) {
if !b.seenPrioFiles[path] {
b.seenPrioFiles[path] = true
b.spec.StargzPrioritizedFiles = append(b.spec.StargzPrioritizedFiles, path)
}
}
func (b *imageSpecBuilder) allocArtifactDir(out builder.BuildOutput) *imageArtifactDir {
hostArtifacts := HostPath(out.GetArtifactDir())
if s := b.seenArtifactDirs[hostArtifacts]; s != nil {
// Already copied this artifact dir.
return s
}
// This artifact directory has not been copied yet.
// Determine a reasonable name for it.
basePath := "/artifacts"
for i := 0; ; i++ {
candidatePath := ImagePath(pathpkg.Join(basePath, strconv.Itoa(i)))
candidate := &imageArtifactDir{
Base: candidatePath,
BuildArtifacts: candidatePath.Join("build"),
}
if b.spec.CopyData[candidate.Base] == "" && b.spec.CopyData[candidate.BuildArtifacts] == "" {
// This name is available.
b.spec.CopyData[candidate.BuildArtifacts] = hostArtifacts
b.seenArtifactDirs[hostArtifacts] = candidate
return candidate
}
// This path already exists. Keep trying.
}
}
func randomProcID() string {
return fmt.Sprintf("proc_%s", xid.New())
}