Skip to content

Commit ca95fec

Browse files
fmeumcopybara-github
fmeum
authored and
copybara-github
committed
Add RBE support for generated unresolved symlinks
Also adds support for such symlinks to the remote worker implementation. Work towards #10298 Closes #15781. PiperOrigin-RevId: 469196671 Change-Id: I441da6d0a658d142153ada6cc0f836bf2ed3bcff
1 parent 56195ef commit ca95fec

File tree

6 files changed

+183
-33
lines changed

6 files changed

+183
-33
lines changed

‎src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java‎

Lines changed: 53 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import build.bazel.remote.execution.v2.Digest;
1717
import com.google.common.base.Preconditions;
18+
import com.google.common.collect.Iterables;
1819
import com.google.common.collect.Sets;
1920
import com.google.devtools.build.lib.vfs.Path;
2021
import com.google.devtools.build.lib.vfs.PathFragment;
@@ -33,7 +34,11 @@
3334
final class DirectoryTree {
3435

3536
interface Visitor {
36-
void visitDirectory(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs);
37+
void visitDirectory(
38+
PathFragment dirname,
39+
List<FileNode> files,
40+
List<SymlinkNode> symlinks,
41+
List<DirectoryNode> dirs);
3742
}
3843

3944
abstract static class Node implements Comparable<Node> {
@@ -138,6 +143,44 @@ public boolean equals(Object o) {
138143
}
139144
return false;
140145
}
146+
147+
@Override
148+
public String toString() {
149+
return String.format(
150+
"%s (hash: %s, size: %d)", getPathSegment(), digest.getHash(), digest.getSizeBytes());
151+
}
152+
}
153+
154+
static class SymlinkNode extends Node {
155+
private final String target;
156+
157+
SymlinkNode(String pathSegment, String target) {
158+
super(pathSegment);
159+
this.target = target;
160+
}
161+
162+
public String getTarget() {
163+
return target;
164+
}
165+
166+
@Override
167+
public int hashCode() {
168+
return Objects.hash(super.hashCode(), target);
169+
}
170+
171+
@Override
172+
public boolean equals(Object o) {
173+
if (o instanceof SymlinkNode) {
174+
SymlinkNode other = (SymlinkNode) o;
175+
return super.equals(other) && Objects.equals(target, other.target);
176+
}
177+
return false;
178+
}
179+
180+
@Override
181+
public String toString() {
182+
return String.format("%s --> %s", getPathSegment(), getTarget());
183+
}
141184
}
142185

143186
static class DirectoryNode extends Node {
@@ -203,10 +246,13 @@ private void visit(Visitor visitor, PathFragment dirname) {
203246
}
204247

205248
List<FileNode> files = new ArrayList<>(dir.children.size());
249+
List<SymlinkNode> symlinks = new ArrayList<>();
206250
List<DirectoryNode> dirs = new ArrayList<>();
207251
for (Node child : dir.children) {
208252
if (child instanceof FileNode) {
209253
files.add((FileNode) child);
254+
} else if (child instanceof SymlinkNode) {
255+
symlinks.add((SymlinkNode) child);
210256
} else if (child instanceof DirectoryNode) {
211257
dirs.add((DirectoryNode) child);
212258
visit(visitor, dirname.getRelative(child.pathSegment));
@@ -215,14 +261,14 @@ private void visit(Visitor visitor, PathFragment dirname) {
215261
String.format("Node type '%s' is not supported", child.getClass().getSimpleName()));
216262
}
217263
}
218-
visitor.visitDirectory(dirname, files, dirs);
264+
visitor.visitDirectory(dirname, files, symlinks, dirs);
219265
}
220266

221267
@Override
222268
public String toString() {
223269
Map<PathFragment, StringBuilder> m = new HashMap<>();
224270
visit(
225-
(dirname, files, dirs) -> {
271+
(dirname, files, symlinks, dirs) -> {
226272
int depth = dirname.segmentCount() - 1;
227273
StringBuilder sb = new StringBuilder();
228274

@@ -231,12 +277,10 @@ public String toString() {
231277
sb.append(dirname.getBaseName());
232278
sb.append("\n");
233279
}
234-
if (!files.isEmpty()) {
235-
for (FileNode file : files) {
236-
sb.append(" ".repeat(2 * (depth + 1)));
237-
sb.append(formatFile(file));
238-
sb.append("\n");
239-
}
280+
for (Node fileOrSymlink : Iterables.concat(files, symlinks)) {
281+
sb.append(" ".repeat(2 * (depth + 1)));
282+
sb.append(fileOrSymlink);
283+
sb.append("\n");
240284
}
241285
if (!dirs.isEmpty()) {
242286
for (DirectoryNode dir : dirs) {
@@ -264,10 +308,4 @@ public boolean equals(Object o) {
264308
DirectoryTree other = (DirectoryTree) o;
265309
return tree.equals(other.tree);
266310
}
267-
268-
private static String formatFile(FileNode file) {
269-
return String.format(
270-
"%s (hash: %s, size: %d)",
271-
file.getPathSegment(), file.digest.getHash(), file.digest.getSizeBytes());
272-
}
273311
}

‎src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,13 @@
1818
import com.google.devtools.build.lib.actions.ActionInput;
1919
import com.google.devtools.build.lib.actions.ActionInputHelper;
2020
import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
21+
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
2122
import com.google.devtools.build.lib.actions.FileArtifactValue;
2223
import com.google.devtools.build.lib.actions.MetadataProvider;
2324
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
2425
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode;
2526
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
27+
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.SymlinkNode;
2628
import com.google.devtools.build.lib.remote.util.DigestUtil;
2729
import com.google.devtools.build.lib.vfs.Dirent;
2830
import com.google.devtools.build.lib.vfs.Path;
@@ -143,11 +145,13 @@ private static int buildFromActionInputs(
143145
input.getExecPathString());
144146
switch (metadata.getType()) {
145147
case REGULAR_FILE:
146-
Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
147-
Path inputPath = ActionInputHelper.toInputPath(input, execRoot);
148-
boolean childAdded =
149-
currDir.addChild(FileNode.createExecutable(path.getBaseName(), inputPath, d));
150-
return childAdded ? 1 : 0;
148+
{
149+
Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
150+
Path inputPath = ActionInputHelper.toInputPath(input, execRoot);
151+
boolean childAdded =
152+
currDir.addChild(FileNode.createExecutable(path.getBaseName(), inputPath, d));
153+
return childAdded ? 1 : 0;
154+
}
151155

152156
case DIRECTORY:
153157
SortedMap<PathFragment, ActionInput> directoryInputs =
@@ -156,11 +160,19 @@ private static int buildFromActionInputs(
156160
directoryInputs, metadataProvider, execRoot, digestUtil, tree);
157161

158162
case SYMLINK:
159-
throw new IllegalStateException(
160-
String.format(
161-
"Encountered symlink input '%s', but all"
162-
+ " symlinks should have been resolved by SkyFrame. This is a bug.",
163-
path));
163+
{
164+
Preconditions.checkState(
165+
input instanceof SpecialArtifact && input.isSymlink(),
166+
"Encountered symlink input '%s', but all source symlinks should have been"
167+
+ " resolved by SkyFrame. This is a bug.",
168+
path);
169+
Path inputPath = ActionInputHelper.toInputPath(input, execRoot);
170+
boolean childAdded =
171+
currDir.addChild(
172+
new SymlinkNode(
173+
path.getBaseName(), inputPath.readSymbolicLink().getPathString()));
174+
return childAdded ? 1 : 0;
175+
}
164176

165177
case SPECIAL_FILE:
166178
throw new IOException(

‎src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import build.bazel.remote.execution.v2.Directory;
2020
import build.bazel.remote.execution.v2.DirectoryNode;
2121
import build.bazel.remote.execution.v2.FileNode;
22+
import build.bazel.remote.execution.v2.SymlinkNode;
2223
import com.google.common.base.Preconditions;
2324
import com.google.common.collect.ArrayListMultimap;
2425
import com.google.common.collect.ImmutableMap;
@@ -94,6 +95,7 @@ private interface MerkleTreeDirectoryVisitor {
9495
@Nullable private final Directory rootProto;
9596
private final Digest rootDigest;
9697
private final SortedSet<DirectoryTree.FileNode> files;
98+
private final SortedSet<DirectoryTree.SymlinkNode> symlinks;
9799
private final SortedMap<String, MerkleTree> directories;
98100
private final long inputFiles;
99101
private final long inputBytes;
@@ -102,6 +104,7 @@ private MerkleTree(
102104
@Nullable Directory rootProto,
103105
Digest rootDigest,
104106
SortedSet<DirectoryTree.FileNode> files,
107+
SortedSet<DirectoryTree.SymlinkNode> symlinks,
105108
SortedMap<String, MerkleTree> directories,
106109
long inputFiles,
107110
long inputBytes) {
@@ -110,6 +113,7 @@ private MerkleTree(
110113
this.rootProto = rootProto;
111114
this.rootDigest = Preconditions.checkNotNull(rootDigest, "rootDigest");
112115
this.files = Preconditions.checkNotNull(files, "files");
116+
this.symlinks = Preconditions.checkNotNull(symlinks, "symlinks");
113117
this.directories = Preconditions.checkNotNull(directories, "directories");
114118
this.inputFiles = inputFiles;
115119
this.inputBytes = inputBytes;
@@ -130,6 +134,10 @@ private SortedSet<DirectoryTree.FileNode> getFiles() {
130134
return files;
131135
}
132136

137+
private SortedSet<DirectoryTree.SymlinkNode> getSymlinks() {
138+
return symlinks;
139+
}
140+
133141
private SortedMap<String, MerkleTree> getDirectories() {
134142
return directories;
135143
}
@@ -243,13 +251,14 @@ private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) {
243251
null,
244252
digestUtil.compute(new byte[0]),
245253
ImmutableSortedSet.of(),
254+
ImmutableSortedSet.of(),
246255
ImmutableSortedMap.of(),
247256
0,
248257
0);
249258
}
250259
Map<PathFragment, MerkleTree> m = new HashMap<>();
251260
tree.visit(
252-
(dirname, files, dirs) -> {
261+
(dirname, files, symlinks, dirs) -> {
253262
SortedMap<String, MerkleTree> subDirs = new TreeMap<>();
254263
for (DirectoryTree.DirectoryNode dir : dirs) {
255264
PathFragment subDirname = dirname.getRelative(dir.getPathSegment());
@@ -258,7 +267,8 @@ private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) {
258267
m.remove(subDirname), "subMerkleTree at '%s' was null", subDirname);
259268
subDirs.put(dir.getPathSegment(), subMerkleTree);
260269
}
261-
MerkleTree mt = buildMerkleTree(new TreeSet<>(files), subDirs, digestUtil);
270+
MerkleTree mt =
271+
buildMerkleTree(new TreeSet<>(files), new TreeSet<>(symlinks), subDirs, digestUtil);
262272
m.put(dirname, mt);
263273
});
264274
MerkleTree rootMerkleTree = m.get(PathFragment.EMPTY_FRAGMENT);
@@ -288,6 +298,10 @@ public static MerkleTree merge(Collection<MerkleTree> merkleTrees, DigestUtil di
288298
for (MerkleTree merkleTree : merkleTrees) {
289299
files.addAll(merkleTree.getFiles());
290300
}
301+
SortedSet<DirectoryTree.SymlinkNode> symlinks = Sets.newTreeSet();
302+
for (MerkleTree merkleTree : merkleTrees) {
303+
symlinks.addAll(merkleTree.getSymlinks());
304+
}
291305

292306
// Group all Merkle trees per path.
293307
Multimap<String, MerkleTree> allDirsToMerge = ArrayListMultimap.create();
@@ -301,24 +315,28 @@ public static MerkleTree merge(Collection<MerkleTree> merkleTrees, DigestUtil di
301315
.forEach(
302316
(baseName, dirsToMerge) -> directories.put(baseName, merge(dirsToMerge, digestUtil)));
303317

304-
return buildMerkleTree(files, directories, digestUtil);
318+
return buildMerkleTree(files, symlinks, directories, digestUtil);
305319
}
306320

307321
private static MerkleTree buildMerkleTree(
308322
SortedSet<DirectoryTree.FileNode> files,
323+
SortedSet<DirectoryTree.SymlinkNode> symlinks,
309324
SortedMap<String, MerkleTree> directories,
310325
DigestUtil digestUtil) {
311326
Directory.Builder b = Directory.newBuilder();
312327
for (DirectoryTree.FileNode file : files) {
313328
b.addFiles(buildProto(file));
314329
}
330+
for (DirectoryTree.SymlinkNode symlink : symlinks) {
331+
b.addSymlinks(buildProto(symlink));
332+
}
315333
for (Map.Entry<String, MerkleTree> nameAndDir : directories.entrySet()) {
316334
b.addDirectories(buildProto(nameAndDir.getKey(), nameAndDir.getValue()));
317335
}
318336
Directory protoDir = b.build();
319337
Digest protoDirDigest = digestUtil.compute(protoDir);
320338

321-
long inputFiles = files.size();
339+
long inputFiles = (long) files.size() + symlinks.size();
322340
for (MerkleTree dir : directories.values()) {
323341
inputFiles += dir.getInputFiles();
324342
}
@@ -331,7 +349,8 @@ private static MerkleTree buildMerkleTree(
331349
inputBytes += dir.getInputBytes();
332350
}
333351

334-
return new MerkleTree(protoDir, protoDirDigest, files, directories, inputFiles, inputBytes);
352+
return new MerkleTree(
353+
protoDir, protoDirDigest, files, symlinks, directories, inputFiles, inputBytes);
335354
}
336355

337356
private static FileNode buildProto(DirectoryTree.FileNode file) {
@@ -349,6 +368,13 @@ private static DirectoryNode buildProto(String baseName, MerkleTree dir) {
349368
.build();
350369
}
351370

371+
private static SymlinkNode buildProto(DirectoryTree.SymlinkNode symlink) {
372+
return SymlinkNode.newBuilder()
373+
.setName(decodeBytestringUtf8(symlink.getPathSegment()))
374+
.setTarget(decodeBytestringUtf8(symlink.getTarget()))
375+
.build();
376+
}
377+
352378
private static PathOrBytes toPathOrBytes(DirectoryTree.FileNode file) {
353379
return file.getPath() != null
354380
? new PathOrBytes(file.getPath())

‎src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode;
2222
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
2323
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.Node;
24+
import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.SymlinkNode;
2425
import com.google.devtools.build.lib.remote.util.DigestUtil;
2526
import com.google.devtools.build.lib.vfs.DigestHashFunction;
2627
import com.google.devtools.build.lib.vfs.FileSystem;
@@ -119,7 +120,10 @@ protected Path createFile(String path, String content) throws IOException {
119120
static void assertLexicographicalOrder(DirectoryTree tree) {
120121
// Assert the lexicographical order as defined by the remote execution protocol
121122
tree.visit(
122-
(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
123+
(PathFragment dirname,
124+
List<FileNode> files,
125+
List<SymlinkNode> symlinks,
126+
List<DirectoryNode> dirs) -> {
123127
assertThat(files).isInStrictOrder();
124128
assertThat(dirs).isInStrictOrder();
125129
});
@@ -136,7 +140,10 @@ private static List<String> asPathSegments(List<? extends Node> nodes) {
136140
private static List<DirectoryNode> directoryNodesAtDepth(DirectoryTree tree, int depth) {
137141
List<DirectoryNode> directoryNodes = new ArrayList<>();
138142
tree.visit(
139-
(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
143+
(PathFragment dirname,
144+
List<FileNode> files,
145+
List<SymlinkNode> symlinks,
146+
List<DirectoryNode> dirs) -> {
140147
int currDepth = dirname.segmentCount();
141148
if (currDepth == depth) {
142149
directoryNodes.addAll(dirs);
@@ -148,7 +155,10 @@ private static List<DirectoryNode> directoryNodesAtDepth(DirectoryTree tree, int
148155
static List<FileNode> fileNodesAtDepth(DirectoryTree tree, int depth) {
149156
List<FileNode> fileNodes = new ArrayList<>();
150157
tree.visit(
151-
(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
158+
(PathFragment dirname,
159+
List<FileNode> files,
160+
List<SymlinkNode> symlinks,
161+
List<DirectoryNode> dirs) -> {
152162
int currDepth = dirname.segmentCount();
153163
if (currDepth == depth) {
154164
fileNodes.addAll(files);

0 commit comments

Comments
 (0)