-
Notifications
You must be signed in to change notification settings - Fork 4k
/
CppCompileAction.java
1890 lines (1728 loc) · 76.9 KB
/
CppCompileAction.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.rules.cpp;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.devtools.build.lib.actions.ActionAnalysisMetadata.mergeMaps;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.io.ByteStreams;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.ActionEnvironment;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.ActionResult;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
import com.google.devtools.build.lib.actions.ArtifactExpander;
import com.google.devtools.build.lib.actions.ArtifactResolver;
import com.google.devtools.build.lib.actions.CommandAction;
import com.google.devtools.build.lib.actions.CommandLine;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.CommandLines.CommandLineAndParamFileInfo;
import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.ParamFileInfo;
import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
import com.google.devtools.build.lib.actions.PathMapper;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SimpleSpawn;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.extra.CppCompileInfo;
import com.google.devtools.build.lib.actions.extra.EnvironmentVariable;
import com.google.devtools.build.lib.actions.extra.ExtraActionInfo;
import com.google.devtools.build.lib.analysis.actions.PathMappers;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.analysis.config.CoreOptions.OutputPathsMode;
import com.google.devtools.build.lib.analysis.starlark.Args;
import com.google.devtools.build.lib.bugreport.BugReport;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.exec.SpawnStrategyResolver;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.ProfilerTask;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.rules.cpp.CcCommon.CoptsFilter;
import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration;
import com.google.devtools.build.lib.rules.cpp.IncludeScanner.IncludeScanningHeaderData;
import com.google.devtools.build.lib.server.FailureDetails.CppCompile;
import com.google.devtools.build.lib.server.FailureDetails.CppCompile.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.skyframe.ActionExecutionValue;
import com.google.devtools.build.lib.starlarkbuildapi.CommandLineArgsApi;
import com.google.devtools.build.lib.util.DependencySet;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.ShellEscaper;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.IORuntimeException;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.SkyframeLookupResult;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Predicate;
import javax.annotation.Nullable;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Sequence;
import net.starlark.java.eval.StarlarkList;
/** Action that represents some kind of C++ compilation step. */
@ThreadCompatible
public class CppCompileAction extends AbstractAction implements IncludeScannable, CommandAction {
private static final UUID GUID = UUID.fromString("97493805-894f-493a-be66-9a698f45c31d");
@VisibleForTesting static final String CPP_COMPILE_MNEMONIC = "CppCompile";
@VisibleForTesting static final String OBJC_COMPILE_MNEMONIC = "ObjcCompile";
@Nullable private final Artifact gcnoFile;
private final Artifact sourceFile;
private final BuildConfigurationValue configuration;
private final NestedSet<Artifact> mandatoryInputs;
private final NestedSet<Artifact> mandatorySpawnInputs;
private final NestedSet<Artifact> allowedDerivedInputs;
/**
* The set of input files that in addition to {@link CcCompilationContext#getDeclaredIncludeSrcs}
* need to be added to the set of input artifacts of the action if we don't use input discovery.
* They may be pruned after execution. See {@link #findUsedHeaders} for more details.
*/
private final NestedSet<Artifact> additionalPrunableHeaders;
@Nullable private final Artifact grepIncludes;
private final boolean shareable;
private final boolean shouldScanIncludes;
private final boolean usePic;
private final boolean useHeaderModules;
private final boolean needsIncludeValidation;
private final CcCompilationContext ccCompilationContext;
private final ImmutableList<Artifact> builtinIncludeFiles;
// A list of files to include scan that are not source files, pcm files, or
// included via a command-line "-include file.h". Actions that use non C++ files as source
// files--such as Clif--may use this mechanism.
private final ImmutableList<Artifact> additionalIncludeScanningRoots;
@VisibleForTesting public final CompileCommandLine compileCommandLine;
/**
* The fingerprint of {@link #compileCommandLine}. This is computed lazily so that the command
* line is not unnecessarily flattened outside of action execution.
*/
private byte[] commandLineKey;
private final ImmutableMap<String, String> executionInfo;
private final String actionName;
private final FeatureConfiguration featureConfiguration;
private final ImmutableList<PathFragment> builtInIncludeDirectories;
// TODO(b/213594908): Make CppCompileAction immutable.
/**
* Set when the action prepares for execution. Used to preserve state between preparation and
* execution.
*/
private NestedSet<Artifact> additionalInputs;
/**
* Used only during input discovery, when input discovery requires other actions to be executed
* first.
*/
private Set<DerivedArtifact> usedModules;
private boolean inputsDiscovered = false;
/**
* This field is set only for C++ module compiles (compiling .cppmap files into .pcm files). It
* stores the modules necessary for building this module as they will later also be required for
* building users of this module. Such users can get to this data through this action's {@link
* com.google.devtools.build.lib.skyframe.ActionExecutionValue}
*
* <p>This field is populated either based on the discovered headers in {@link #discoverInputs} or
* extracted from the action inputs when restoring it from the action cache.
*/
private NestedSet<Artifact> discoveredModules = null;
/** Used modules that are not transitively used through other topLevelModules. */
private NestedSet<Artifact> topLevelModules;
private ParamFileActionInput paramFileActionInput;
private PathFragment paramFilePath;
/**
* Creates a new action to compile C/C++ source files.
*
* @param owner the owner of the action, usually the configured target that emitted it
* @param featureConfiguration TODO(bazel-team): Add parameter description.
* @param variables TODO(bazel-team): Add parameter description.
* @param sourceFile the source file that should be compiled. {@code mandatoryInputs} must contain
* this file
* @param shouldScanIncludes a boolean indicating whether scanning of {@code sourceFile} is to be
* performed looking for inclusions.
* @param usePic TODO(bazel-team): Add parameter description.
* @param mandatoryInputs any additional files that need to be present for the compilation to
* succeed, can be empty but not null, for example, extra sources for FDO.
* @param outputFile the object file that is written as result of the compilation
* @param dotdFile the .d file that is generated as a side-effect of compilation
* @param diagnosticsFile the .dia file that is generated as a side-effect of compilation
* @param gcnoFile the coverage notes that are written in coverage mode, can be null
* @param dwoFile the .dwo output file where debug information is stored for Fission builds (null
* if Fission mode is disabled)
* @param ccCompilationContext the {@code CcCompilationContext}
* @param coptsFilter regular expression to remove options from {@code copts}
* @param additionalIncludeScanningRoots list of additional artifacts to include-scan
* @param actionName a string giving the name of this action for the purpose of toolchain
* evaluation
* @param cppSemantics C++ compilation semantics
* @param builtInIncludeDirectories - list of toolchain-defined builtin include directories.
*/
CppCompileAction(
ActionOwner owner,
FeatureConfiguration featureConfiguration,
CcToolchainVariables variables,
Artifact sourceFile,
BuildConfigurationValue configuration,
boolean shareable,
boolean shouldScanIncludes,
boolean usePic,
boolean useHeaderModules,
NestedSet<Artifact> mandatoryInputs,
NestedSet<Artifact> mandatorySpawnInputs,
ImmutableList<Artifact> builtinIncludeFiles,
NestedSet<Artifact> additionalPrunableHeaders,
Artifact outputFile,
@Nullable Artifact dotdFile,
@Nullable Artifact diagnosticsFile,
@Nullable Artifact gcnoFile,
@Nullable Artifact dwoFile,
@Nullable Artifact ltoIndexingFile,
CcCompilationContext ccCompilationContext,
CoptsFilter coptsFilter,
ImmutableList<Artifact> additionalIncludeScanningRoots,
ImmutableMap<String, String> executionInfo,
String actionName,
CppSemantics cppSemantics,
ImmutableList<PathFragment> builtInIncludeDirectories,
@Nullable Artifact grepIncludes,
ImmutableList<Artifact> additionalOutputs) {
super(
owner,
mandatoryInputs,
collectOutputs(
Preconditions.checkNotNull(outputFile, "outputFile"),
dotdFile,
diagnosticsFile,
gcnoFile,
dwoFile,
ltoIndexingFile,
additionalOutputs));
this.gcnoFile = gcnoFile;
this.sourceFile = sourceFile;
this.shareable = shareable;
this.configuration = configuration;
// We do not need to include the middleman artifact since it is a generated artifact and will
// definitely exist prior to this action execution.
this.mandatoryInputs = mandatoryInputs;
this.mandatorySpawnInputs = mandatorySpawnInputs;
this.additionalPrunableHeaders = additionalPrunableHeaders;
this.shouldScanIncludes = shouldScanIncludes && cppSemantics.allowIncludeScanning();
this.usePic = usePic;
this.useHeaderModules = useHeaderModules;
this.ccCompilationContext = ccCompilationContext;
this.builtinIncludeFiles = builtinIncludeFiles;
this.additionalIncludeScanningRoots =
Preconditions.checkNotNull(additionalIncludeScanningRoots);
this.compileCommandLine =
buildCommandLine(
sourceFile,
coptsFilter,
actionName,
dotdFile,
featureConfiguration,
variables);
this.executionInfo = executionInfo;
this.actionName = actionName;
this.featureConfiguration = featureConfiguration;
this.needsIncludeValidation = cppSemantics.needsIncludeValidation();
this.builtInIncludeDirectories = builtInIncludeDirectories;
this.additionalInputs = null;
this.usedModules = null;
this.topLevelModules = null;
this.grepIncludes = grepIncludes;
if (featureConfiguration.isEnabled(CppRuleClasses.COMPILER_PARAM_FILE)) {
paramFilePath =
outputFile
.getExecPath()
.getParentDirectory()
.getChild(outputFile.getFilename() + ".params");
}
NestedSetBuilder<Artifact> allowedDerivedInputsBuilder =
NestedSetBuilder.fromNestedSet(mandatoryInputs)
.addTransitive(additionalPrunableHeaders)
.addTransitive(ccCompilationContext.getTransitiveCompilationPrerequisites())
.addTransitive(ccCompilationContext.getDeclaredIncludeSrcs())
.addTransitive(ccCompilationContext.getTransitiveModules(usePic))
.add(getSourceFile());
// The separate module is an allowed input to all compiles of this context except for its own
// compile.
Artifact separateModule = ccCompilationContext.getSeparateHeaderModule(usePic);
if (separateModule != null && !separateModule.equals(getPrimaryOutput())) {
allowedDerivedInputsBuilder.add(separateModule);
}
allowedDerivedInputs = allowedDerivedInputsBuilder.build();
}
private static ImmutableSet<Artifact> collectOutputs(
Artifact outputFile,
@Nullable Artifact dotdFile,
@Nullable Artifact diagnosticsFile,
@Nullable Artifact gcnoFile,
@Nullable Artifact dwoFile,
@Nullable Artifact ltoIndexingFile,
ImmutableList<Artifact> additionalOutputs) {
ImmutableSet.Builder<Artifact> outputs = ImmutableSet.builder();
outputs.add(outputFile);
if (gcnoFile != null) {
outputs.add(gcnoFile);
}
outputs.addAll(additionalOutputs);
if (dotdFile != null) {
outputs.add(dotdFile);
}
if (diagnosticsFile != null) {
outputs.add(diagnosticsFile);
}
if (dwoFile != null) {
outputs.add(dwoFile);
}
if (ltoIndexingFile != null) {
outputs.add(ltoIndexingFile);
}
return outputs.build();
}
static CompileCommandLine buildCommandLine(
Artifact sourceFile,
CoptsFilter coptsFilter,
String actionName,
Artifact dotdFile,
FeatureConfiguration featureConfiguration,
CcToolchainVariables variables) {
return CompileCommandLine.builder(sourceFile, coptsFilter, actionName, dotdFile)
.setFeatureConfiguration(featureConfiguration)
.setVariables(variables)
.build();
}
/**
* Whether we should do "include scanning". Note that this does *not* mean whether we should parse
* the .d files to determine which include files were used during compilation. Instead, this means
* whether we should a) run the pre-execution include scanner (see {@code IncludeScanningContext})
* if one exists and b) whether the action inputs should be modified to match the results of that
* pre-execution scanning and (if enabled) again after execution to match the results of the .d
* file parsing.
*
* <p>This does *not* have anything to do with "hdrs_check".
*/
@VisibleForTesting
boolean shouldScanIncludes() {
return shouldScanIncludes;
}
boolean useInMemoryDotdFiles() {
return cppConfiguration().getInmemoryDotdFiles();
}
private boolean enabledCppCompileResourcesEstimation() {
return cppConfiguration().getExperimentalCppCompileResourcesEstimation();
}
@Override
public ActionEnvironment getEnvironment() {
return configuration.getActionEnvironment();
}
@Override
public List<PathFragment> getBuiltInIncludeDirectories() {
return builtInIncludeDirectories;
}
@Nullable
@Override
public List<Artifact> getBuiltInIncludeFiles() {
return builtinIncludeFiles;
}
@Override
public NestedSet<Artifact> getMandatoryInputs() {
return mandatoryInputs;
}
@Override
public ImmutableSet<Artifact> getMandatoryOutputs() {
// Never prune orphaned modules files. To cut down critical paths, CppCompileActions do not
// add modules files as inputs. Instead they rely on input discovery to recognize the needed
// ones. However, orphan detection runs before input discovery and thus module files would be
// discarded as orphans.
// This is strictly better than marking all transitive modules as inputs, which would also
// effectively disable orphan detection for .pcm files.
Artifact outputFile = getPrimaryOutput();
if (outputFile.isFileType(CppFileTypes.CPP_MODULE)) {
return ImmutableSet.of(outputFile);
}
return super.getMandatoryOutputs();
}
/**
* Returns the list of additional inputs found by dependency discovery, during action preparation.
* {@link #discoverInputs(ActionExecutionContext)} must be called before this method is called on
* each action execution.
*/
public NestedSet<Artifact> getAdditionalInputs() {
return Preconditions.checkNotNull(additionalInputs);
}
/** Clears the discovered {@link #additionalInputs}. */
private void clearAdditionalInputs() {
additionalInputs = null;
}
@Override
public boolean discoversInputs() {
return shouldScanIncludes || getDotdFile() != null || shouldParseShowIncludes();
}
@Override
protected boolean inputsDiscovered() {
return inputsDiscovered;
}
@Override
protected void setInputsDiscovered(boolean inputsDiscovered) {
this.inputsDiscovered = inputsDiscovered;
}
@Override
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public NestedSet<Artifact> getPossibleInputsForTesting() {
return NestedSetBuilder.fromNestedSet(getInputs())
.addTransitive(ccCompilationContext.getDeclaredIncludeSrcs())
.addTransitive(additionalPrunableHeaders)
.build();
}
/** Returns the results of include scanning. */
@Nullable
private NestedSet<Artifact> findUsedHeaders(
ActionExecutionContext actionExecutionContext, IncludeScanningHeaderData headerData)
throws ActionExecutionException, InterruptedException {
Preconditions.checkState(
shouldScanIncludes, "findUsedHeaders() called although include scanning is disabled");
try {
try {
List<Artifact> includes =
actionExecutionContext
.getContext(CppIncludeScanningContext.class)
.findAdditionalInputs(this, actionExecutionContext, headerData);
if (includes == null) {
return null;
}
Collections.sort(includes, Artifact.EXEC_PATH_COMPARATOR);
return NestedSetBuilder.wrap(Order.STABLE_ORDER, includes);
} catch (IORuntimeException e) {
throw new EnvironmentalExecException(
e.getCauseIOException(),
createFailureDetail("Find used headers failure", Code.FIND_USED_HEADERS_IO_EXCEPTION));
}
} catch (ExecException e) {
throw ActionExecutionException.fromExecException(e, "include scanning", this);
}
}
// TODO(b/213594908): Remove this method from Action interface once CppCompileAction is immutable.
@Override
public void prepareInputDiscovery() {
// Make sure to clear the additional inputs potentially left over from an old build (in case we
// ran discoverInputs, but not beginExecution).
clearAdditionalInputs();
}
/**
* This method returns null when a required SkyValue is missing and a Skyframe restart is
* required.
*/
@Nullable
@Override
public NestedSet<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
Preconditions.checkArgument(!sourceFile.isFileType(CppFileTypes.CPP_MODULE));
if (additionalInputs == null) {
List<String> options;
try {
options = getCompilerOptions();
} catch (CommandLineExpansionException e) {
String message =
String.format(
"failed to generate compile command for rule '%s: %s",
getOwner().getLabel(), e.getMessage());
DetailedExitCode code = createDetailedExitCode(message, Code.COMMAND_GENERATION_FAILURE);
throw new ActionExecutionException(message, this, /*catastrophe=*/ false, code);
}
commandLineKey = computeCommandLineKey(options);
ImmutableList<PathFragment> systemIncludeDirs = getSystemIncludeDirs(options);
boolean siblingLayout =
actionExecutionContext
.getOptions()
.getOptions(BuildLanguageOptions.class)
.experimentalSiblingRepositoryLayout;
if (!shouldScanIncludes) {
// When not actually doing include scanning, add all prunable headers to additionalInputs.
// This is necessary because the inputs that can be pruned by .d file parsing must be
// returned from discoverInputs() and they cannot be in mandatoryInputs. Thus, even with
// include scanning turned off, we pretend that we "discover" these headers.
additionalInputs =
NestedSetBuilder.fromNestedSet(ccCompilationContext.getDeclaredIncludeSrcs())
.addTransitive(additionalPrunableHeaders)
.build();
if (needsIncludeValidation) {
verifyActionIncludePaths(systemIncludeDirs, siblingLayout);
}
return additionalInputs;
}
IncludeScanningHeaderData.Builder includeScanningHeaderDataBuilder =
ccCompilationContext.createIncludeScanningHeaderData(
actionExecutionContext.getEnvironmentForDiscoveringInputs(),
usePic,
useHeaderModules);
if (includeScanningHeaderDataBuilder == null) {
return null;
}
// In theory, we could verify include paths even earlier, but we want to avoid the restart
// above necessitating a double-execution.
if (needsIncludeValidation) {
verifyActionIncludePaths(systemIncludeDirs, siblingLayout);
}
IncludeScanningHeaderData includeScanningHeaderData =
includeScanningHeaderDataBuilder
.setSystemIncludeDirs(systemIncludeDirs)
.setCmdlineIncludes(getCmdlineIncludes(options))
.setIsValidUndeclaredHeader(getValidUndeclaredHeaderPredicate())
.build();
additionalInputs = findUsedHeaders(actionExecutionContext, includeScanningHeaderData);
if (additionalInputs == null) {
return null;
}
if (useHeaderModules) {
boolean separate =
getPrimaryOutput().equals(ccCompilationContext.getSeparateHeaderModule(usePic));
usedModules =
ccCompilationContext.computeUsedModules(usePic, additionalInputs.toSet(), separate);
}
}
if (usedModules == null) {
// There are two paths in which this can be reached:
// 1. This is not a modular compilation or one without include scanning. In either case, we
// never compute used modules.
// 2. This function has completed on a previous execution, adding all used modules to
// additionalInputs and resetting usedModules to null below.
// In either case, there is nothing more to do here.
return additionalInputs;
}
ImmutableMap<Artifact, NestedSet<Artifact>> transitivelyUsedModules =
computeTransitivelyUsedModules(
actionExecutionContext.getEnvironmentForDiscoveringInputs(), usedModules);
if (transitivelyUsedModules == null) {
return null;
}
Set<Artifact> topLevel =
actionExecutionContext
.getDiscoveredModulesPruner()
.computeTopLevelModules(this, usedModules, transitivelyUsedModules);
NestedSetBuilder<Artifact> topLevelModulesBuilder = NestedSetBuilder.stableOrder();
NestedSetBuilder<Artifact> discoveredModulesBuilder = NestedSetBuilder.stableOrder();
for (Artifact module : topLevel) {
topLevelModulesBuilder.add(module);
discoveredModulesBuilder.addTransitive(transitivelyUsedModules.get(module));
}
topLevelModules = topLevelModulesBuilder.build();
discoveredModulesBuilder.addTransitive(topLevelModules);
NestedSet<Artifact> discoveredModules = discoveredModulesBuilder.buildInterruptibly();
additionalInputs =
NestedSetBuilder.fromNestedSet(additionalInputs).addTransitive(discoveredModules).build();
if (getPrimaryOutput().isFileType(CppFileTypes.CPP_MODULE)) {
this.discoveredModules = discoveredModules;
}
usedModules = null;
return additionalInputs;
}
@Override
protected final NestedSet<Artifact> getOriginalInputs() {
return mandatoryInputs;
}
@Nullable
private Predicate<Artifact> getValidUndeclaredHeaderPredicate() {
if (getDotdFile() != null) {
// If we'll be looking at .d files later, don't remove undeclared inputs now.
return null;
}
var cppConfiguration = cppConfiguration();
Iterable<PathFragment> ignoreDirs =
cppConfiguration.isStrictSystemIncludes()
? getBuiltInIncludeDirectories()
: getValidationIgnoredDirs();
ImmutableSet<Artifact> additionalPrunableHeadersSet = additionalPrunableHeaders.toSet();
return header ->
additionalPrunableHeadersSet.contains(header)
|| FileSystemUtils.startsWithAny(header.getExecPath(), ignoreDirs);
}
@Override
public Artifact getPrimaryInput() {
return getSourceFile();
}
/** Returns the path of the c/cc source for gcc. */
public final Artifact getSourceFile() {
return compileCommandLine.getSourceFile();
}
@Override
@Nullable
public Artifact getGrepIncludes() {
return grepIncludes;
}
/**
* Set by {@link #discoverInputs}. Returns a subset of {@link #getAdditionalInputs} or an empty
* {@link NestedSet}, if this is not a compile action producing a C++ module.
*/
@Override
public NestedSet<Artifact> getDiscoveredModules() {
return firstNonNull(discoveredModules, NestedSetBuilder.emptySet(Order.STABLE_ORDER));
}
/** Returns the path where the compiler should put the discovered dependency information. */
public Artifact getDotdFile() {
return compileCommandLine.getDotdFile();
}
public CcCompilationContext getCcCompilationContext() {
return ccCompilationContext;
}
@Override
public List<PathFragment> getQuoteIncludeDirs() {
ImmutableList.Builder<PathFragment> result = ImmutableList.builder();
result.addAll(ccCompilationContext.getQuoteIncludeDirs());
ImmutableList<String> copts = compileCommandLine.getCopts(PathMapper.NOOP);
for (int i = 0; i < copts.size(); i++) {
String opt = copts.get(i);
if (opt.startsWith("-iquote")) {
if (opt.length() > 7) {
result.add(PathFragment.create(opt.substring(7).trim()));
} else if (i + 1 < copts.size()) {
i++;
result.add(PathFragment.create(copts.get(i)));
} else {
System.err.println("WARNING: dangling -iquote flag in options for " + prettyPrint());
}
}
}
return result.build();
}
@Override
public List<PathFragment> getIncludeDirs() {
ImmutableList.Builder<PathFragment> result = ImmutableList.builder();
result.addAll(ccCompilationContext.getIncludeDirs());
for (String opt : compileCommandLine.getCopts(PathMapper.NOOP)) {
if (opt.startsWith("-I") || opt.startsWith("/I")) {
// We insist on the combined form "-Idir".
String includeDir = opt.substring(2);
if (includeDir.isEmpty()) {
continue;
}
if (matchesCaseInsensitiveMsvc(includeDir)) {
// This is actually a "-imsvc", a system include dir.
continue;
}
result.add(PathFragment.create(opt.substring(2)));
}
}
return result.build();
}
@Override
public ImmutableList<PathFragment> getFrameworkIncludeDirs() {
return ccCompilationContext.getFrameworkIncludeDirs();
}
@VisibleForTesting
List<PathFragment> getSystemIncludeDirs() throws CommandLineExpansionException {
return getSystemIncludeDirs(getCompilerOptions());
}
private ImmutableList<PathFragment> getSystemIncludeDirs(List<String> compilerOptions) {
// TODO(bazel-team): parsing the command line flags here couples us to gcc- and clang-cl-style
// compiler command lines; use a different way to specify system includes (for example through a
// system_includes attribute in cc_toolchain); note that that would disallow users from
// specifying system include paths via the copts attribute.
// Currently, this works together with the include_paths features because getCommandLine() will
// get the system include paths from the {@code CcCompilationContext} instead.
ImmutableList.Builder<PathFragment> result = ImmutableList.builder();
for (int i = 0; i < compilerOptions.size(); i++) {
String opt = compilerOptions.get(i);
String systemIncludeFlag = null;
if (opt.startsWith("-isystem")) {
systemIncludeFlag = "-isystem";
} else if (matchesIncludeCaseInsensitiveMsvc(opt)) {
systemIncludeFlag = opt.substring(0, 6);
}
if (systemIncludeFlag == null) {
continue;
}
if (opt.length() > systemIncludeFlag.length()) {
result.add(PathFragment.create(opt.substring(systemIncludeFlag.length()).trim()));
} else if (i + 1 < compilerOptions.size()) {
i++;
result.add(PathFragment.create(compilerOptions.get(i)));
} else {
System.err.println(
"WARNING: dangling " + systemIncludeFlag + " flag in options for " + prettyPrint());
}
}
return result.build();
}
private CppConfiguration cppConfiguration() {
return configuration.getFragment(CppConfiguration.class);
}
private static final ImmutableList<CharMatcher> MSVC_CHARS =
ImmutableList.of(
CharMatcher.anyOf("mM"),
CharMatcher.anyOf("sS"),
CharMatcher.anyOf("vV"),
CharMatcher.anyOf("cC"));
private static final ImmutableList<CharMatcher> INCLUDE_PREFIX_CHARS =
ImmutableList.of(CharMatcher.anyOf("-/"), CharMatcher.anyOf("iI"));
private static boolean substrMatchesChars(
String s, int startPos, ImmutableList<CharMatcher> substr) {
for (int i = 0; i < substr.size(); i++) {
if (!substr.get(i).matches(s.charAt(startPos + i))) {
return false;
}
}
return true;
}
private static boolean matchesCaseInsensitiveMsvc(String s) {
return s.length() >= 4 && substrMatchesChars(s, 0, MSVC_CHARS);
}
private static boolean matchesIncludeCaseInsensitiveMsvc(String s) {
return s.length() >= 6
&& substrMatchesChars(s, 0, INCLUDE_PREFIX_CHARS)
&& substrMatchesChars(s, 2, MSVC_CHARS);
}
private static ImmutableList<String> getCmdlineIncludes(List<String> args) {
ImmutableList.Builder<String> cmdlineIncludes = ImmutableList.builder();
for (Iterator<String> argi = args.iterator(); argi.hasNext(); ) {
String arg = argi.next();
if (arg.equals("-include") && argi.hasNext()) {
cmdlineIncludes.add(argi.next());
} else if (arg.startsWith("-FI") || arg.startsWith("/FI")) {
if (arg.length() > 3) {
cmdlineIncludes.add(arg.substring(3).trim());
} else if (argi.hasNext()) {
cmdlineIncludes.add(argi.next());
}
}
}
return cmdlineIncludes.build();
}
@Override
public Artifact getMainIncludeScannerSource() {
// getIncludeScannerSources() needs to return the main file first. This is used for determining
// what file command line includes should be interpreted relative to.
return getIncludeScannerSources().get(0);
}
@SuppressWarnings("LenientFormatStringValidation")
@Override
public ImmutableList<Artifact> getIncludeScannerSources() {
if (getSourceFile().isFileType(CppFileTypes.CPP_MODULE_MAP)) {
Artifact outputFile = getPrimaryOutput();
boolean isSeparate = outputFile.equals(ccCompilationContext.getSeparateHeaderModule(usePic));
// Expected 0 args, but got 1.
Preconditions.checkState(
outputFile.equals(ccCompilationContext.getHeaderModule(usePic)) || isSeparate,
"Trying to build unknown module",
outputFile);
// If this is an action that compiles the header module itself, the source we build is the
// module map, and we need to include-scan all headers that are referenced in the module map.
return ccCompilationContext.getHeaderModuleSrcs(isSeparate);
}
ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
builder.add(getSourceFile());
builder.addAll(additionalIncludeScanningRoots);
return builder.build();
}
/**
* Returns the list of "-D" arguments that should be used by this gcc invocation. Only used for
* testing.
*/
@VisibleForTesting
public ImmutableCollection<String> getDefines() {
return ccCompilationContext.getDefines();
}
@Override
@VisibleForTesting
public ImmutableMap<String, String> getIncompleteEnvironmentForTesting()
throws ActionExecutionException {
try {
return getEffectiveEnvironment(ImmutableMap.of());
} catch (CommandLineExpansionException e) {
String message =
String.format(
"failed to generate compile environment variables for rule '%s: %s",
getOwner().getLabel(), e.getMessage());
DetailedExitCode code = createDetailedExitCode(message, Code.COMMAND_GENERATION_FAILURE);
throw new ActionExecutionException(message, this, /*catastrophe=*/ false, code);
}
}
@Override
public ImmutableMap<String, String> getEffectiveEnvironment(Map<String, String> clientEnv)
throws CommandLineExpansionException {
return getEffectiveEnvironment(clientEnv, PathMapper.NOOP);
}
public ImmutableMap<String, String> getEffectiveEnvironment(
Map<String, String> clientEnv, PathMapper pathMapper) throws CommandLineExpansionException {
ActionEnvironment env = getEnvironment();
Map<String, String> environment = Maps.newLinkedHashMapWithExpectedSize(env.estimatedSize());
env.resolve(environment, clientEnv);
if (!getExecutionInfo().containsKey(ExecutionRequirements.REQUIRES_DARWIN)) {
// Linux: this prevents gcc/clang from writing the unpredictable (and often irrelevant) value
// of getcwd() into the debug info. Not applicable to Darwin or Windows, which have no /proc.
environment.put("PWD", "/proc/self/cwd");
}
environment.putAll(compileCommandLine.getEnvironment(pathMapper));
return ImmutableMap.copyOf(environment);
}
@Override
public List<String> getArguments() throws CommandLineExpansionException {
return getArguments(PathMapper.NOOP);
}
private List<String> getArguments(PathMapper pathMapper) throws CommandLineExpansionException {
return compileCommandLine.getArguments(paramFilePath, getOverwrittenVariables(), pathMapper);
}
@Override
public Sequence<String> getStarlarkArgv() throws EvalException {
try {
return StarlarkList.immutableCopyOf(
compileCommandLine.getArguments(
/* parameterFilePath= */ null, getOverwrittenVariables(), PathMapper.NOOP));
} catch (CommandLineExpansionException ex) {
throw new EvalException(ex);
}
}
@Override
public Sequence<CommandLineArgsApi> getStarlarkArgs() {
ImmutableSet<Artifact> directoryInputs =
getInputs().toList().stream().filter(Artifact::isDirectory).collect(toImmutableSet());
CommandLine commandLine = compileCommandLine.getFilteredFeatureConfigurationCommandLine(this);
ParamFileInfo paramFileInfo = null;
if (cppConfiguration().useArgsParamsFile()) {
paramFileInfo =
ParamFileInfo.builder(ParameterFileType.GCC_QUOTED)
.setCharset(ISO_8859_1)
.setUseAlways(true)
.build();
}
CommandLineAndParamFileInfo commandLineAndParamFileInfo =
new CommandLineAndParamFileInfo(commandLine, paramFileInfo);
Args args = Args.forRegisteredAction(commandLineAndParamFileInfo, directoryInputs);
return StarlarkList.immutableCopyOf(ImmutableList.of(args));
}
@Override
public ExtraActionInfo.Builder getExtraActionInfo(ActionKeyContext actionKeyContext)
throws CommandLineExpansionException, InterruptedException {
CppCompileInfo.Builder info = CppCompileInfo.newBuilder();
info.setTool(compileCommandLine.getToolPath());
List<String> options =
compileCommandLine.getCompilerOptions(getOverwrittenVariables(), PathMapper.NOOP);
for (String option : options) {
info.addCompilerOption(option);
}
info.setOutputFile(getPrimaryOutput().getExecPathString());
info.setSourceFile(getSourceFile().getExecPathString());
if (inputsKnown()) {
info.addAllSourcesAndHeaders(Artifact.toExecPaths(getInputs().toList()));
} else {
info.addSourcesAndHeaders(getSourceFile().getExecPathString());
info.addAllSourcesAndHeaders(
Artifact.toExecPaths(ccCompilationContext.getDeclaredIncludeSrcs().toList()));
}
// TODO(ulfjack): Extra actions currently ignore the client environment.
for (Map.Entry<String, String> envVariable :
getEffectiveEnvironment(/*clientEnv=*/ ImmutableMap.of()).entrySet()) {
info.addVariable(
EnvironmentVariable.newBuilder()
.setName(envVariable.getKey())
.setValue(envVariable.getValue())
.build());
}
try {
return super.getExtraActionInfo(actionKeyContext)
.setExtension(CppCompileInfo.cppCompileInfo, info.build());
} catch (CommandLineExpansionException e) {
throw new AssertionError("CppCompileAction command line expansion cannot fail", e);
}
}
/** Returns the compiler options. */
@VisibleForTesting
public List<String> getCompilerOptions() throws CommandLineExpansionException {
return compileCommandLine.getCompilerOptions(/* overwrittenVariables= */ null, PathMapper.NOOP);
}
@Override
public ImmutableMap<String, String> getExecutionInfo() {
return mergeMaps(super.getExecutionInfo(), executionInfo);
}
private static boolean validateInclude(
Set<Artifact> allowedIncludes, Iterable<PathFragment> ignoreDirs, Artifact include) {
// Only declared modules are added to an action and so they are always valid.
return include.isFileType(CppFileTypes.CPP_MODULE)
||
// TODO(b/145253507): Exclude objc module maps from check, due to bad interaction with
// local_objc_modules feature.
include.isFileType(CppFileTypes.OBJC_MODULE_MAP)
||
// It's a declared include/
allowedIncludes.contains(include)
||
// Ignore headers from built-in include directories.
FileSystemUtils.startsWithAny(include.getExecPath(), ignoreDirs);
}
/**
* Enforce that the includes actually visited during the compile were properly declared in the
* rules.
*
* <p>The technique is to walk through all of the reported includes that gcc emits into the .d
* file, and verify that they came from acceptable relative include directories. This is done in
* two steps: