-
Notifications
You must be signed in to change notification settings - Fork 9.5k
/
module_install.go
884 lines (790 loc) · 35.1 KB
/
module_install.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package initwd
import (
"context"
"errors"
"fmt"
"log"
"os"
"path"
"path/filepath"
"strings"
"github.com/apparentlymart/go-versions/versions"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/configs"
"github.com/hashicorp/terraform/internal/configs/configload"
"github.com/hashicorp/terraform/internal/getmodules"
"github.com/hashicorp/terraform/internal/modsdir"
"github.com/hashicorp/terraform/internal/registry"
"github.com/hashicorp/terraform/internal/registry/regsrc"
"github.com/hashicorp/terraform/internal/registry/response"
"github.com/hashicorp/terraform/internal/tfdiags"
)
type ModuleInstaller struct {
modsDir string
loader *configload.Loader
reg *registry.Client
// The keys in moduleVersions are resolved and trimmed registry source
// addresses and the values are the registry response.
registryPackageVersions map[addrs.ModuleRegistryPackage]*response.ModuleVersions
// The keys in moduleVersionsUrl are the moduleVersion struct below and
// addresses and the values are underlying remote source addresses.
registryPackageSources map[moduleVersion]addrs.ModuleSourceRemote
}
type moduleVersion struct {
module addrs.ModuleRegistryPackage
version string
}
func NewModuleInstaller(modsDir string, loader *configload.Loader, reg *registry.Client) *ModuleInstaller {
return &ModuleInstaller{
modsDir: modsDir,
loader: loader,
reg: reg,
registryPackageVersions: make(map[addrs.ModuleRegistryPackage]*response.ModuleVersions),
registryPackageSources: make(map[moduleVersion]addrs.ModuleSourceRemote),
}
}
// InstallModules analyses the root module in the given directory and installs
// all of its direct and transitive dependencies into the given modules
// directory, which must already exist.
//
// Since InstallModules makes possibly-time-consuming calls to remote services,
// a hook interface is supported to allow the caller to be notified when
// each module is installed and, for remote modules, when downloading begins.
// LoadConfig guarantees that two hook calls will not happen concurrently but
// it does not guarantee any particular ordering of hook calls. This mechanism
// is for UI feedback only and does not give the caller any control over the
// process.
//
// If modules are already installed in the target directory, they will be
// skipped unless their source address or version have changed or unless
// the upgrade flag is set.
//
// InstallModules never deletes any directory, except in the case where it
// needs to replace a directory that is already present with a newly-extracted
// package.
//
// If the returned diagnostics contains errors then the module installation
// may have wholly or partially completed. Modules must be loaded in order
// to find their dependencies, so this function does many of the same checks
// as LoadConfig as a side-effect.
//
// If successful (the returned diagnostics contains no errors) then the
// first return value is the early configuration tree that was constructed by
// the installation process.
func (i *ModuleInstaller) InstallModules(ctx context.Context, rootDir string, upgrade bool, hooks ModuleInstallHooks) (*configs.Config, tfdiags.Diagnostics) {
log.Printf("[TRACE] ModuleInstaller: installing child modules for %s into %s", rootDir, i.modsDir)
var diags tfdiags.Diagnostics
rootMod, mDiags := i.loader.Parser().LoadConfigDir(rootDir)
if rootMod == nil {
// We drop the diagnostics here because we only want to report module
// loading errors after checking the core version constraints, which we
// can only do if the module can be at least partially loaded.
return nil, diags
} else if vDiags := rootMod.CheckCoreVersionRequirements(nil, nil); vDiags.HasErrors() {
// If the core version requirements are not met, we drop any other
// diagnostics, as they may reflect language changes from future
// Terraform versions.
diags = diags.Append(vDiags)
} else {
diags = diags.Append(mDiags)
}
manifest, err := modsdir.ReadManifestSnapshotForDir(i.modsDir)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to read modules manifest file",
fmt.Sprintf("Error reading manifest for %s: %s.", i.modsDir, err),
))
return nil, diags
}
fetcher := getmodules.NewPackageFetcher()
cfg, instDiags := i.installDescendentModules(ctx, rootMod, rootDir, manifest, upgrade, hooks, fetcher)
diags = append(diags, instDiags...)
return cfg, diags
}
func (i *ModuleInstaller) installDescendentModules(ctx context.Context, rootMod *configs.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks ModuleInstallHooks, fetcher *getmodules.PackageFetcher) (*configs.Config, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
if hooks == nil {
// Use our no-op implementation as a placeholder
hooks = ModuleInstallHooksImpl{}
}
// Create a manifest record for the root module. This will be used if
// there are any relative-pathed modules in the root.
manifest[""] = modsdir.Record{
Key: "",
Dir: rootDir,
}
cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
var diags hcl.Diagnostics
if req.SourceAddr == nil {
// If the parent module failed to parse the module source
// address, we can't load it here. Return nothing as the parent
// module's diagnostics should explain this.
return nil, nil, diags
}
if req.Name == "" {
// An empty string for a module instance name breaks our
// manifest map, which uses that to indicate the root module.
// Because we descend into modules which have errors, we need
// to look out for this case, but the config loader's
// diagnostics will report the error later.
return nil, nil, diags
}
key := manifest.ModuleKey(req.Path)
instPath := i.packageInstallPath(req.Path)
log.Printf("[DEBUG] Module installer: begin %s", key)
// First we'll check if we need to upgrade/replace an existing
// installed module, and delete it out of the way if so.
replace := upgrade
if !replace {
record, recorded := manifest[key]
switch {
case !recorded:
log.Printf("[TRACE] ModuleInstaller: %s is not yet installed", key)
replace = true
case record.SourceAddr != req.SourceAddr.String():
log.Printf("[TRACE] ModuleInstaller: %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
replace = true
case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
log.Printf("[TRACE] ModuleInstaller: %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
replace = true
}
}
// If we _are_ planning to replace this module, then we'll remove
// it now so our installation code below won't conflict with any
// existing remnants.
if replace {
if _, recorded := manifest[key]; recorded {
log.Printf("[TRACE] ModuleInstaller: discarding previous record of %s prior to reinstall", key)
}
delete(manifest, key)
// Deleting a module invalidates all of its descendent modules too.
keyPrefix := key + "."
for subKey := range manifest {
if strings.HasPrefix(subKey, keyPrefix) {
if _, recorded := manifest[subKey]; recorded {
log.Printf("[TRACE] ModuleInstaller: also discarding downstream %s", subKey)
}
delete(manifest, subKey)
}
}
}
record, recorded := manifest[key]
if !recorded {
// Clean up any stale cache directory that might be present.
// If this is a local (relative) source then the dir will
// not exist, but we'll ignore that.
log.Printf("[TRACE] ModuleInstaller: cleaning directory %s prior to install of %s", instPath, key)
err := os.RemoveAll(instPath)
if err != nil && !os.IsNotExist(err) {
log.Printf("[TRACE] ModuleInstaller: failed to remove %s: %s", key, err)
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to remove local module cache",
Detail: fmt.Sprintf(
"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
instPath, err,
),
})
return nil, nil, diags
}
} else {
// If this module is already recorded and its root directory
// exists then we will just load what's already there and
// keep our existing record.
info, err := os.Stat(record.Dir)
if err == nil && info.IsDir() {
mod, mDiags := i.loader.Parser().LoadConfigDir(record.Dir)
if mod == nil {
// nil indicates an unreadable module, which should never happen,
// so we return the full loader diagnostics here.
diags = diags.Extend(mDiags)
} else if vDiags := mod.CheckCoreVersionRequirements(req.Path, req.SourceAddr); vDiags.HasErrors() {
// If the core version requirements are not met, we drop any other
// diagnostics, as they may reflect language changes from future
// Terraform versions.
diags = diags.Extend(vDiags)
} else {
diags = diags.Extend(mDiags)
}
log.Printf("[TRACE] ModuleInstaller: Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
return mod, record.Version, diags
}
}
// If we get down here then it's finally time to actually install
// the module. There are some variants to this process depending
// on what type of module source address we have.
switch addr := req.SourceAddr.(type) {
case addrs.ModuleSourceLocal:
log.Printf("[TRACE] ModuleInstaller: %s has local path %q", key, addr.String())
mod, mDiags := i.installLocalModule(req, key, manifest, hooks)
mDiags = maybeImproveLocalInstallError(req, mDiags)
diags = append(diags, mDiags...)
return mod, nil, diags
case addrs.ModuleSourceRegistry:
log.Printf("[TRACE] ModuleInstaller: %s is a registry module at %s", key, addr.String())
mod, v, mDiags := i.installRegistryModule(ctx, req, key, instPath, addr, manifest, hooks, fetcher)
diags = append(diags, mDiags...)
return mod, v, diags
case addrs.ModuleSourceRemote:
log.Printf("[TRACE] ModuleInstaller: %s address %q will be handled by go-getter", key, addr.String())
mod, mDiags := i.installGoGetterModule(ctx, req, key, instPath, manifest, hooks, fetcher)
diags = append(diags, mDiags...)
return mod, nil, diags
default:
// Shouldn't get here, because there are no other implementations
// of addrs.ModuleSource.
panic(fmt.Sprintf("unsupported module source address %#v", addr))
}
},
))
diags = diags.Append(cDiags)
err := manifest.WriteSnapshotToDir(i.modsDir)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to update module manifest",
fmt.Sprintf("Unable to write the module manifest file: %s", err),
))
}
return cfg, diags
}
func (i *ModuleInstaller) installLocalModule(req *configs.ModuleRequest, key string, manifest modsdir.Manifest, hooks ModuleInstallHooks) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
parentKey := manifest.ModuleKey(req.Parent.Path)
parentRecord, recorded := manifest[parentKey]
if !recorded {
// This is indicative of a bug rather than a user-actionable error
panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
}
if len(req.VersionConstraint.Required) != 0 {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid version constraint",
Detail: fmt.Sprintf("Cannot apply a version constraint to module %q (at %s:%d) because it has a relative local path.", req.Name, req.CallRange.Filename, req.CallRange.Start.Line),
Subject: req.CallRange.Ptr(),
})
}
// For local sources we don't actually need to modify the
// filesystem at all because the parent already wrote
// the files we need, and so we just load up what's already here.
newDir := filepath.Join(parentRecord.Dir, req.SourceAddr.String())
log.Printf("[TRACE] ModuleInstaller: %s uses directory from parent: %s", key, newDir)
// it is possible that the local directory is a symlink
newDir, err := filepath.EvalSymlinks(newDir)
if err != nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("Unable to evaluate directory symlink: %s", err.Error()),
})
}
// Finally we are ready to try actually loading the module.
mod, mDiags := i.loader.Parser().LoadConfigDir(newDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read for module %q at %s:%d.", newDir, req.Name, req.CallRange.Filename, req.CallRange.Start.Line),
})
} else if vDiags := mod.CheckCoreVersionRequirements(req.Path, req.SourceAddr); vDiags.HasErrors() {
// If the core version requirements are not met, we drop any other
// diagnostics, as they may reflect language changes from future
// Terraform versions.
diags = diags.Extend(vDiags)
} else {
diags = diags.Extend(mDiags)
}
// Note the local location in our manifest.
manifest[key] = modsdir.Record{
Key: key,
Dir: newDir,
SourceAddr: req.SourceAddr.String(),
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
hooks.Install(key, nil, newDir)
return mod, diags
}
func (i *ModuleInstaller) installRegistryModule(ctx context.Context, req *configs.ModuleRequest, key string, instPath string, addr addrs.ModuleSourceRegistry, manifest modsdir.Manifest, hooks ModuleInstallHooks, fetcher *getmodules.PackageFetcher) (*configs.Module, *version.Version, hcl.Diagnostics) {
var diags hcl.Diagnostics
hostname := addr.Package.Host
reg := i.reg
var resp *response.ModuleVersions
var exists bool
// A registry entry isn't _really_ a module package, but we'll pretend it's
// one for the sake of this reporting by just trimming off any source
// directory.
packageAddr := addr.Package
// Our registry client is still using the legacy model of addresses, so
// we'll shim it here for now.
regsrcAddr := regsrc.ModuleFromRegistryPackageAddr(packageAddr)
// check if we've already looked up this module from the registry
if resp, exists = i.registryPackageVersions[packageAddr]; exists {
log.Printf("[TRACE] %s using already found available versions of %s at %s", key, addr, hostname)
} else {
var err error
log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
resp, err = reg.ModuleVersions(ctx, regsrcAddr)
if err != nil {
if registry.IsModuleNotFound(err) {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module not found",
Detail: fmt.Sprintf("Module %q (from %s:%d) cannot be found in the module registry at %s.", req.Name, req.CallRange.Filename, req.CallRange.Start.Line, hostname),
Subject: req.CallRange.Ptr(),
})
} else if errors.Is(err, context.Canceled) {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module installation was interrupted",
Detail: fmt.Sprintf("Received interrupt signal while retrieving available versions for module %q.", req.Name),
})
} else {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Error accessing remote module registry",
Detail: fmt.Sprintf("Failed to retrieve available versions for module %q (%s:%d) from %s: %s.", req.Name, req.CallRange.Filename, req.CallRange.Start.Line, hostname, err),
Subject: req.CallRange.Ptr(),
})
}
return nil, nil, diags
}
i.registryPackageVersions[packageAddr] = resp
}
// The response might contain information about dependencies to allow us
// to potentially optimize future requests, but we don't currently do that
// and so for now we'll just take the first item which is guaranteed to
// be the address we requested.
if len(resp.Modules) < 1 {
// Should never happen, but since this is a remote service that may
// be implemented by third-parties we will handle it gracefully.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for module %q (%s:%d).", hostname, req.Name, req.CallRange.Filename, req.CallRange.Start.Line),
Subject: req.CallRange.Ptr(),
})
return nil, nil, diags
}
modMeta := resp.Modules[0]
var latestMatch *version.Version
var latestVersion *version.Version
for _, mv := range modMeta.Versions {
v, err := version.NewVersion(mv.Version)
if err != nil {
// Should never happen if the registry server is compliant with
// the protocol, but we'll warn if not to assist someone who
// might be developing a module registry server.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagWarning,
Summary: "Invalid response from remote module registry",
Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for module %q (%s:%d), which Terraform ignored.", hostname, mv.Version, req.Name, req.CallRange.Filename, req.CallRange.Start.Line),
Subject: req.CallRange.Ptr(),
})
continue
}
// If we've found a pre-release version then we'll ignore it unless
// it was exactly requested.
//
// The prerelease checking will be handled by a different library for
// 2 reasons. First, this other library automatically includes the
// "prerelease versions must be exactly requested" behaviour that we are
// looking for. Second, this other library is used to handle all version
// constraints for the provider logic and this is the first step to
// making the module and provider version logic match.
if v.Prerelease() != "" {
// At this point all versions published by the module with
// prerelease metadata will be checked. Users may not have even
// requested this prerelease so don't print lots of unnecessary #
// warnings.
acceptableVersions, err := versions.MeetingConstraintsString(req.VersionConstraint.Required.String())
if err != nil {
log.Printf("[WARN] ModuleInstaller: %s ignoring %s because the version constraints (%s) could not be parsed: %s", key, v, req.VersionConstraint.Required.String(), err.Error())
continue
}
// Validate the version is also readable by the other versions
// library.
version, err := versions.ParseVersion(v.String())
if err != nil {
log.Printf("[WARN] ModuleInstaller: %s ignoring %s because the version (%s) reported by the module could not be parsed: %s", key, v, v.String(), err.Error())
continue
}
// Finally, check if the prerelease is acceptable to version. As
// highlighted previously, we go through all of this because the
// apparentlymart/go-versions library handles prerelease constraints
// in the apporach we want to.
if !acceptableVersions.Has(version) {
log.Printf("[TRACE] ModuleInstaller: %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
continue
}
// If we reach here, it means this prerelease version was exactly
// requested according to the extra constraints of this library.
// We fall through and allow the other library to also validate it
// for consistency.
}
if latestVersion == nil || v.GreaterThan(latestVersion) {
latestVersion = v
}
if req.VersionConstraint.Required.Check(v) {
if latestMatch == nil || v.GreaterThan(latestMatch) {
latestMatch = v
}
}
}
if latestVersion == nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module has no versions",
Detail: fmt.Sprintf("Module %q (%s:%d) has no versions available on %s.", addr, req.CallRange.Filename, req.CallRange.Start.Line, hostname),
Subject: req.CallRange.Ptr(),
})
return nil, nil, diags
}
if latestMatch == nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unresolvable module version constraint",
Detail: fmt.Sprintf("There is no available version of module %q (%s:%d) which matches the given version constraint. The newest available version is %s.", addr, req.CallRange.Filename, req.CallRange.Start.Line, latestVersion),
Subject: req.CallRange.Ptr(),
})
return nil, nil, diags
}
// Report up to the caller that we're about to start downloading.
hooks.Download(key, packageAddr.String(), latestMatch)
// If we manage to get down here then we've found a suitable version to
// install, so we need to ask the registry where we should download it from.
// The response to this is a go-getter-style address string.
// first check the cache for the download URL
moduleAddr := moduleVersion{module: packageAddr, version: latestMatch.String()}
if _, exists := i.registryPackageSources[moduleAddr]; !exists {
realAddrRaw, err := reg.ModuleLocation(ctx, regsrcAddr, latestMatch.String())
if err != nil {
log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Error accessing remote module registry",
Detail: fmt.Sprintf("Failed to retrieve a download URL for %s %s from %s: %s", addr, latestMatch, hostname, err),
})
return nil, nil, diags
}
realAddr, err := addrs.ParseModuleSource(realAddrRaw)
if err != nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid package location from module registry",
Detail: fmt.Sprintf("Module registry %s returned invalid source location %q for %s %s: %s.", hostname, realAddrRaw, addr, latestMatch, err),
})
return nil, nil, diags
}
switch realAddr := realAddr.(type) {
// Only a remote source address is allowed here: a registry isn't
// allowed to return a local path (because it doesn't know what
// its being called from) and we also don't allow recursively pointing
// at another registry source for simplicity's sake.
case addrs.ModuleSourceRemote:
i.registryPackageSources[moduleAddr] = realAddr
default:
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid package location from module registry",
Detail: fmt.Sprintf("Module registry %s returned invalid source location %q for %s %s: must be a direct remote package address.", hostname, realAddrRaw, addr, latestMatch),
})
return nil, nil, diags
}
}
dlAddr := i.registryPackageSources[moduleAddr]
log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, packageAddr, latestMatch, dlAddr.Package)
err := fetcher.FetchPackage(ctx, instPath, dlAddr.Package.String())
if errors.Is(err, context.Canceled) {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module download was interrupted",
Detail: fmt.Sprintf("Interrupt signal received when downloading module %s.", addr),
})
return nil, nil, diags
}
if err != nil {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s.", req.Name, req.CallRange.Filename, req.CallRange.Start.Line, dlAddr, err),
Subject: req.CallRange.Ptr(),
})
return nil, nil, diags
}
log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, dlAddr.Package, instPath)
// Incorporate any subdir information from the original path into the
// address returned by the registry in order to find the final directory
// of the target module.
finalAddr := dlAddr.FromRegistry(addr)
subDir := filepath.FromSlash(finalAddr.Subdir)
modDir := filepath.Join(instPath, subDir)
log.Printf("[TRACE] ModuleInstaller: %s should now be at %s", key, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := i.loader.Parser().LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For registry modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
})
} else if vDiags := mod.CheckCoreVersionRequirements(req.Path, req.SourceAddr); vDiags.HasErrors() {
// If the core version requirements are not met, we drop any other
// diagnostics, as they may reflect language changes from future
// Terraform versions.
diags = diags.Extend(vDiags)
} else {
diags = diags.Extend(mDiags)
}
// Note the local location in our manifest.
manifest[key] = modsdir.Record{
Key: key,
Version: latestMatch,
Dir: modDir,
SourceAddr: req.SourceAddr.String(),
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, latestMatch, modDir)
return mod, latestMatch, diags
}
func (i *ModuleInstaller) installGoGetterModule(ctx context.Context, req *configs.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks ModuleInstallHooks, fetcher *getmodules.PackageFetcher) (*configs.Module, hcl.Diagnostics) {
var diags hcl.Diagnostics
// Report up to the caller that we're about to start downloading.
addr := req.SourceAddr.(addrs.ModuleSourceRemote)
packageAddr := addr.Package
hooks.Download(key, packageAddr.String(), nil)
if len(req.VersionConstraint.Required) != 0 {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Invalid version constraint",
Detail: fmt.Sprintf("Cannot apply a version constraint to module %q (at %s:%d) because it doesn't come from a module registry.", req.Name, req.CallRange.Filename, req.CallRange.Start.Line),
Subject: req.CallRange.Ptr(),
})
return nil, diags
}
err := fetcher.FetchPackage(ctx, instPath, packageAddr.String())
if err != nil {
// go-getter generates a poor error for an invalid relative path, so
// we'll detect that case and generate a better one.
if _, ok := err.(*getmodules.MaybeRelativePathErr); ok {
log.Printf(
"[TRACE] ModuleInstaller: %s looks like a local path but is missing ./ or ../",
req.SourceAddr,
)
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Module not found",
Detail: fmt.Sprintf(
"The module address %q could not be resolved.\n\n"+
"If you intended this as a path relative to the current "+
"module, use \"./%s\" instead. The \"./\" prefix "+
"indicates that the address is a relative filesystem path.",
req.SourceAddr, req.SourceAddr,
),
})
} else {
// Errors returned by go-getter have very inconsistent quality as
// end-user error messages, but for now we're accepting that because
// we have no way to recognize any specific errors to improve them
// and masking the error entirely would hide valuable diagnostic
// information from the user.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to download module",
Detail: fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s", req.Name, req.CallRange.Filename, req.CallRange.Start.Line, packageAddr, err),
Subject: req.CallRange.Ptr(),
})
}
return nil, diags
}
modDir, err := getmodules.ExpandSubdirGlobs(instPath, addr.Subdir)
if err != nil {
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Failed to expand subdir globs",
Detail: err.Error(),
})
return nil, diags
}
log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, addr, modDir)
// Finally we are ready to try actually loading the module.
mod, mDiags := i.loader.Parser().LoadConfigDir(modDir)
if mod == nil {
// nil indicates missing or unreadable directory, so we'll
// discard the returned diags and return a more specific
// error message here. For go-getter modules this actually
// indicates a bug in the code above, since it's not the
// user's responsibility to create the directory in this case.
diags = diags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unreadable module directory",
Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
})
} else if vDiags := mod.CheckCoreVersionRequirements(req.Path, req.SourceAddr); vDiags.HasErrors() {
// If the core version requirements are not met, we drop any other
// diagnostics, as they may reflect language changes from future
// Terraform versions.
diags = diags.Extend(vDiags)
} else {
diags = diags.Extend(mDiags)
}
// Note the local location in our manifest.
manifest[key] = modsdir.Record{
Key: key,
Dir: modDir,
SourceAddr: req.SourceAddr.String(),
}
log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
hooks.Install(key, nil, modDir)
return mod, diags
}
func (i *ModuleInstaller) packageInstallPath(modulePath addrs.Module) string {
return filepath.Join(i.modsDir, strings.Join(modulePath, "."))
}
// maybeImproveLocalInstallError is a helper function which can recognize
// some specific situations where it can return a more helpful error message
// and thus replace the given errors with those if so.
//
// If this function can't do anything about a particular situation then it
// will just return the given diags verbatim.
//
// This function's behavior is only reasonable for errors returned from the
// ModuleInstaller.installLocalModule function.
func maybeImproveLocalInstallError(req *configs.ModuleRequest, diags hcl.Diagnostics) hcl.Diagnostics {
if !diags.HasErrors() {
return diags
}
// The main situation we're interested in detecting here is whether the
// current module or any of its ancestors use relative paths that reach
// outside of the "package" established by the nearest non-local ancestor.
// That's never really valid, but unfortunately we historically didn't
// have any explicit checking for it and so now for compatibility in
// situations where things just happened to "work" we treat this as an
// error only in situations where installation would've failed anyway,
// so we can give a better error about it than just a generic
// "directory not found" or whatever.
//
// Since it's never actually valid to relative out of the containing
// package, we just assume that any failed local package install which
// does so was caused by that, because to stop doing it should always
// improve the situation, even if it leads to another error describing
// a different problem.
// To decide this we need to find the subset of our ancestors that
// belong to the same "package" as our request, along with the closest
// ancestor that defined that package, and then we can work forwards
// to see if any of the local paths "escaped" the package.
type Step struct {
Path addrs.Module
SourceAddr addrs.ModuleSource
}
var packageDefiner Step
var localRefs []Step
localRefs = append(localRefs, Step{
Path: req.Path,
SourceAddr: req.SourceAddr,
})
current := req.Parent // a configs.Config where Children isn't populated yet
for {
if current == nil || current.SourceAddr == nil {
// We've reached the root module, in which case we aren't
// in an external "package" at all and so our special case
// can't apply.
return diags
}
if _, ok := current.SourceAddr.(addrs.ModuleSourceLocal); !ok {
// We've found the package definer, then!
packageDefiner = Step{
Path: current.Path,
SourceAddr: current.SourceAddr,
}
break
}
localRefs = append(localRefs, Step{
Path: current.Path,
SourceAddr: current.SourceAddr,
})
current = current.Parent
}
// Our localRefs list is reversed because we were traversing up the tree,
// so we'll flip it the other way and thus walk "downwards" through it.
for i, j := 0, len(localRefs)-1; i < j; i, j = i+1, j-1 {
localRefs[i], localRefs[j] = localRefs[j], localRefs[i]
}
// Our method here is to start with a known base path prefix and
// then apply each of the local refs to it in sequence until one of
// them causes us to "lose" the prefix. If that happens, we've found
// an escape to report. This is not an exact science but good enough
// heuristic for choosing a better error message.
const prefix = "*/" // NOTE: this can find a false negative if the user chooses "*" as a directory name, but we consider that unlikely
packageAddr, startPath := splitAddrSubdir(packageDefiner.SourceAddr)
currentPath := path.Join(prefix, startPath)
for _, step := range localRefs {
rel := step.SourceAddr.String()
nextPath := path.Join(currentPath, rel)
if !strings.HasPrefix(nextPath, prefix) { // ESCAPED!
escapeeAddr := step.Path.String()
var newDiags hcl.Diagnostics
// First we'll copy over any non-error diagnostics from the source diags
for _, diag := range diags {
if diag.Severity != hcl.DiagError {
newDiags = newDiags.Append(diag)
}
}
// ...but we'll replace any errors with this more precise error.
var suggestion string
if strings.HasPrefix(packageAddr, "/") || filepath.VolumeName(packageAddr) != "" {
// It might be somewhat surprising that Terraform treats
// absolute filesystem paths as "external" even though it
// treats relative paths as local, so if it seems like that's
// what the user was doing then we'll add an additional note
// about it.
suggestion = "\n\nTerraform treats absolute filesystem paths as external modules which establish a new module package. To treat this directory as part of the same package as its caller, use a local path starting with either \"./\" or \"../\"."
}
newDiags = newDiags.Append(&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Local module path escapes module package",
Detail: fmt.Sprintf(
"The given source directory for %s would be outside of its containing package %q. Local source addresses starting with \"../\" must stay within the same package that the calling module belongs to.%s",
escapeeAddr, packageAddr, suggestion,
),
})
return newDiags
}
currentPath = nextPath
}
// If we get down here then we have nothing useful to do, so we'll just
// echo back what we were given.
return diags
}
func splitAddrSubdir(addr addrs.ModuleSource) (string, string) {
switch addr := addr.(type) {
case addrs.ModuleSourceRegistry:
subDir := addr.Subdir
addr.Subdir = ""
return addr.String(), subDir
case addrs.ModuleSourceRemote:
return addr.Package.String(), addr.Subdir
case nil:
panic("splitAddrSubdir on nil addrs.ModuleSource")
default:
return addr.String(), ""
}
}