Skip to content
Permalink
Browse files

Allow a SpawnRunner to inject output metadata.

Adds a MetadataInjector type that can be accessed via the
SpawnExecutionContext.

Progress towards bazelbuild#6862
  • Loading branch information...
buchgr committed Jan 28, 2019
1 parent 81f3566 commit fbf284db58fde4dd4ed55ef3d00352fc4a03a971
@@ -31,7 +31,7 @@
* <p>Note that implementations of this interface call chmod on output files if {@link
* #discardOutputMetadata} has been called.
*/
public interface MetadataHandler extends MetadataProvider {
public interface MetadataHandler extends MetadataProvider, MetadataInjector {
@Override
FileArtifactValue getMetadata(ActionInput actionInput) throws IOException;

@@ -56,9 +56,6 @@
*/
void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest);

/** Injects a file that is only stored remotely. */
void injectRemoteFile(Artifact output, byte[] digest, long size, int locationIndex);

/**
* Marks an artifact as intentionally omitted. Acknowledges that this Artifact could have existed,
* but was intentionally not saved, most likely as an optimization.
@@ -0,0 +1,49 @@
// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.actions.cache;

import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Map;

/**
* Allows to inject metadata of action outputs into skyframe.
*/
public interface MetadataInjector {

/**
* Injects metadata of a file that is stored remotely.
*
* @param file a regular output file.
* @param digest the digest of the file.
* @param sizeBytes the size of the file in bytes.
* @param locationIndex is only used in Blaze.
*/
default void injectRemoteFile(Artifact file, byte[] digest, long sizeBytes, int locationIndex) {
throw new UnsupportedOperationException();
}

/**
* Inject the metadata of a tree artifact whose contents are stored remotely.
*
* @param treeArtifact an output directory.
* @param children the metadata of the files stored in the directory.
*/
default void injectRemoteDirectory(SpecialArtifact treeArtifact,
Map<PathFragment, RemoteFileArtifactValue> children) {
throw new UnsupportedOperationException();
}
}
@@ -32,6 +32,7 @@
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
@@ -193,6 +194,11 @@ public void prefetchInputs() throws IOException {
public MetadataProvider getMetadataProvider() {
return actionExecutionContext.getMetadataProvider();
}

@Override
public MetadataHandler getMetadataInjector() {
return actionExecutionContext.getMetadataHandler();
}

@Override
public ArtifactExpander getArtifactExpander() {
@@ -21,6 +21,7 @@
import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.cache.MetadataInjector;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.IOException;
@@ -199,6 +200,14 @@ default ArtifactPathResolver getPathResolver() {

/** Reports a progress update to the Spawn strategy. */
void report(ProgressStatus state, String name);

/**
* Returns a {@link MetadataInjector} that allows a caller to inject metadata about spawn
* outputs that are stored remotely.
*/
default MetadataInjector getMetadataInjector() {
throw new UnsupportedOperationException();
}
}

/**
@@ -16,6 +16,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -29,6 +30,7 @@
import com.google.devtools.build.lib.actions.ArtifactFileMetadata;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.actions.FileStateValue;
import com.google.devtools.build.lib.actions.cache.Md5Digest;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
@@ -479,6 +481,18 @@ public void injectRemoteFile(Artifact output, byte[] digest, long size, int loca
store.injectRemoteFile(output, digest, size, locationIndex);
}

@Override
public void injectRemoteDirectory(SpecialArtifact output,
Map<PathFragment, RemoteFileArtifactValue> children) {
ImmutableMap.Builder<TreeFileArtifact, FileArtifactValue> childFileValues = ImmutableMap.builder();
for (Map.Entry<PathFragment, RemoteFileArtifactValue> child : children.entrySet()) {
childFileValues.put(ActionInputHelper.treeFileArtifact(output, child.getKey()), child.getValue());
}

TreeArtifactValue treeArtifactValue = TreeArtifactValue.create(childFileValues.build());
store.putTreeArtifactData(output, treeArtifactValue);
}

@Override
public void markOmitted(ActionInput output) {
Preconditions.checkState(executionMode.get());
@@ -17,6 +17,7 @@
import static org.junit.Assert.fail;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.ActionInputMap;
@@ -28,10 +29,12 @@
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.testutil.Scratch;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import java.io.FileNotFoundException;
import java.util.Map;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -248,4 +251,66 @@ public void withUnknownOutputArtifactMissingDisallowedTreeArtifact() throws Exce
} catch (IllegalStateException expected) {
}
}

@Test
public void injectRemoteArtifactMetadata() throws Exception {
PathFragment path = PathFragment.create("foo/bar");
Artifact artifact = new Artifact(path, outputRoot);
ActionMetadataHandler handler = new ActionMetadataHandler(
/* inputArtifactData= */ new ActionInputMap(0),
/* missingArtifactsAllowed= */ false,
/* outputs= */ ImmutableList.of(artifact),
/* tsgm= */ null,
ArtifactPathResolver.IDENTITY,
new OutputStore());
handler.discardOutputMetadata();

byte[] digest = new byte[]{1,2,3};
int size = 10;
handler.injectRemoteFile(artifact, digest, size, /* locationIndex= */ 1);

FileArtifactValue v = handler.getMetadata(artifact);
assertThat(v).isNotNull();
assertThat(v.getDigest()).isEqualTo(digest);
assertThat(v.getSize()).isEqualTo(size);
assertThat(v.isRemote()).isTrue();
}

@Test
public void injectRemoteTreeArtifactMetadata() throws Exception {
PathFragment path = PathFragment.create("bin/dir");
SpecialArtifact treeArtifact =
new SpecialArtifact(
outputRoot, path, ArtifactOwner.NullArtifactOwner.INSTANCE, SpecialArtifactType.TREE);
OutputStore store = new OutputStore();
ActionMetadataHandler handler = new ActionMetadataHandler(
/* inputArtifactData= */ new ActionInputMap(0),
/* missingArtifactsAllowed= */ false,
/* outputs= */ ImmutableList.of(treeArtifact),
/* tsgm= */ null,
ArtifactPathResolver.IDENTITY,
store);
handler.discardOutputMetadata();

RemoteFileArtifactValue fooValue = new RemoteFileArtifactValue(new byte[]{1,2,3}, 5, 1);
RemoteFileArtifactValue barValue = new RemoteFileArtifactValue(new byte[]{4,5,6}, 10, 1);
Map<PathFragment, RemoteFileArtifactValue> children =
ImmutableMap.<PathFragment, RemoteFileArtifactValue>builder()
.put(PathFragment.create("foo"), fooValue)
.put(PathFragment.create("bar"), barValue)
.build();

handler.injectRemoteDirectory(treeArtifact, children);

FileArtifactValue value = handler.getMetadata(treeArtifact);
assertThat(value).isNotNull();
TreeArtifactValue treeValue = store.getTreeArtifactData(treeArtifact);
assertThat(treeValue).isNotNull();
assertThat(treeValue.getDigest()).isEqualTo(value.getDigest());

assertThat(treeValue.getChildPaths()).containsExactly(PathFragment.create("foo"),
PathFragment.create("bar"));
assertThat(treeValue.getChildValues().values()).containsExactly(fooValue, barValue);
assertThat(treeValue.isRemote()).isTrue();
}
}

0 comments on commit fbf284d

Please sign in to comment.
You can’t perform that action at this time.