Skip to content

Commit

Permalink
Memoize feature merging
Browse files Browse the repository at this point in the history 
Potentially fixed #42
  • Loading branch information
@kolloch
kolloch committed Nov 27, 2019
1 parent f7d1a4e commit ec8aab5
Show file tree
Hide file tree
Showing 11 changed files with 450 additions and 437 deletions.
82 changes: 44 additions & 38 deletions crate2nix/Cargo.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3179,31 +3179,25 @@ rec {
combined;
in builtins.toJSON { inherit onlyInCargo onlyInCrate2Nix differentFeatures; };

/* Returns the feature configuration by package id for the given input crate. */
mergePackageFeatures = {
crateConfigs ? crates,
packageId,
features ? rootFeatures,
...
} @ args:
assert (builtins.isAttrs crateConfigs);
assert (builtins.isString packageId);
assert (builtins.isList features);

let packageFeatures = listOfPackageFeatures args;
grouped = lib.groupBy (x: x.packageId) packageFeatures;
in lib.mapAttrs (n: v: sortedUnique (builtins.concatLists (builtins.map (v: v.features) v))) grouped;
/* Returns the feature configuration by package id for the given input crate.
/* Returns a { packageId, features } attribute set for every package needed for building the
Returns a { packageId, features } attribute set for every package needed for building the
package for the given packageId with the given features.
Returns multiple, potentially conflicting attribute sets for dependencies that are reachable
by multiple paths in the dependency tree.
*/
listOfPackageFeatures = {crateConfigs ? crates, packageId, features, dependencyPath? [packageId], ...} @ args:
mergePackageFeatures = {
crateConfigs ? crates,
packageId,
features ? rootFeatures,
dependencyPath? [crates.${packageId}.crateName],
featuresByPackageId? {},
...} @ args:
assert (builtins.isAttrs crateConfigs);
assert (builtins.isString packageId);
assert (builtins.isList features);
assert (builtins.isAttrs featuresByPackageId);

let
crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}");
Expand All @@ -3214,26 +3208,41 @@ rec {
features = dependencyFeatures expandedFeatures dependency;
in { inherit packageId features; };

resolveDependencies = path: dependencies:
resolveDependencies = cache: path: dependencies:
assert (builtins.isAttrs cache);
assert (builtins.isList dependencies);

let enabledDependencies = filterEnabledDependencies dependencies expandedFeatures;
directDependencies = map depWithResolvedFeatures enabledDependencies;
in builtins.concatMap
({packageId, features}: listOfPackageFeatures {
# This is purely for debugging.
dependencyPath = dependencyPath ++ [path packageId];
inherit crateConfigs packageId features;
})
directDependencies;
foldOverCache = op: lib.foldl op cache directDependencies;
in foldOverCache
(cache: {packageId, features}:
let cacheFeatures = cache.${packageId} or [];
combinedFeatures = sortedUnique (cacheFeatures ++ features);
in
if cache ? ${packageId} && cache.${packageId} == combinedFeatures
then cache
else mergePackageFeatures {
# This is purely for debugging.
dependencyPath = dependencyPath ++ [path crateConfigs.${packageId}.crateName];
features = combinedFeatures;
featuresByPackageId = cache;
inherit crateConfigs packageId;
});

cacheWithSelf =
let cacheFeatures = featuresByPackageId.${packageId} or [];
combinedFeatures = sortedUnique (cacheFeatures ++ expandedFeatures);
in featuresByPackageId // {
${packageId} = combinedFeatures;
};

resolvedDependencies = builtins.concatLists
[
(resolveDependencies "dependencies" (crateConfig.dependencies or []))
(resolveDependencies "buildDependencies" (crateConfig.buildDependencies or []))
];
cacheWithDependencies =
resolveDependencies cacheWithSelf "dep" (crateConfig.dependencies or []);
cacheWithAll =
resolveDependencies cacheWithDependencies "build" (crateConfig.buildDependencies or []);

in [{inherit packageId; features = expandedFeatures;}] ++ resolvedDependencies;
in cacheWithAll;

/* Returns the enabled dependencies given the enabled features. */
filterEnabledDependencies = dependencies: features:
Expand All @@ -3243,10 +3252,8 @@ rec {
lib.filter
(dep:
let targetFunc = dep.target or (features: true);
target = targetFunc features;
in builtins.isString dep
|| target
&& (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep.name) features))
in targetFunc features
&& (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep.name) features))
dependencies;

/* Returns whether the given feature should enable the given dependency. */
Expand Down Expand Up @@ -3278,14 +3285,13 @@ rec {
*/
dependencyFeatures = features: dependency:
assert (builtins.isList features);
assert (builtins.isAttrs dependency || builtins.isString dependency);
assert (builtins.isAttrs dependency);

let defaultOrNil = if builtins.isString dependency || dependency.usesDefaultFeatures or true
let defaultOrNil = if dependency.usesDefaultFeatures or true
then ["default"]
else [];
explicitFeatures = if builtins.isString dependency then [] else dependency.features or [];
explicitFeatures = dependency.features or [];
additionalDependencyFeatures =

let dependencyPrefix = dependency.name+"/";
dependencyFeatures =
builtins.filter (f: lib.hasPrefix dependencyPrefix f) features;
Expand Down
86 changes: 46 additions & 40 deletions crate2nix/templates/nix/crate2nix/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,31 +180,25 @@ rec {
combined;
in builtins.toJSON { inherit onlyInCargo onlyInCrate2Nix differentFeatures; };

/* Returns the feature configuration by package id for the given input crate. */
mergePackageFeatures = {
crateConfigs ? crates,
packageId,
features ? rootFeatures,
...
} @ args:
assert (builtins.isAttrs crateConfigs);
assert (builtins.isString packageId);
assert (builtins.isList features);

let packageFeatures = listOfPackageFeatures args;
grouped = lib.groupBy (x: x.packageId) packageFeatures;
in lib.mapAttrs (n: v: sortedUnique (builtins.concatLists (builtins.map (v: v.features) v))) grouped;
/* Returns the feature configuration by package id for the given input crate.
/* Returns a { packageId, features } attribute set for every package needed for building the
Returns a { packageId, features } attribute set for every package needed for building the
package for the given packageId with the given features.
Returns multiple, potentially conflicting attribute sets for dependencies that are reachable
by multiple paths in the dependency tree.
*/
listOfPackageFeatures = {crateConfigs ? crates, packageId, features, dependencyPath? [packageId], ...} @ args:
mergePackageFeatures = {
crateConfigs ? crates,
packageId,
features ? rootFeatures,
dependencyPath? [crates.${packageId}.crateName],
featuresByPackageId? {},
...} @ args:
assert (builtins.isAttrs crateConfigs);
assert (builtins.isString packageId);
assert (builtins.isList features);
assert (builtins.isAttrs featuresByPackageId);

let
crateConfig = crateConfigs."${packageId}" or (builtins.throw "Package not found: ${packageId}");
Expand All @@ -215,26 +209,41 @@ rec {
features = dependencyFeatures expandedFeatures dependency;
in { inherit packageId features; };

resolveDependencies = path: dependencies:
resolveDependencies = cache: path: dependencies:
assert (builtins.isAttrs cache);
assert (builtins.isList dependencies);

let enabledDependencies = filterEnabledDependencies dependencies expandedFeatures;
directDependencies = map depWithResolvedFeatures enabledDependencies;
in builtins.concatMap
({packageId, features}: listOfPackageFeatures {
# This is purely for debugging.
dependencyPath = dependencyPath ++ [path packageId];
inherit crateConfigs packageId features;
})
directDependencies;

resolvedDependencies = builtins.concatLists
[
(resolveDependencies "dependencies" (crateConfig.dependencies or []))
(resolveDependencies "buildDependencies" (crateConfig.buildDependencies or []))
];

in [{inherit packageId; features = expandedFeatures;}] ++ resolvedDependencies;
foldOverCache = op: lib.foldl op cache directDependencies;
in foldOverCache
(cache: {packageId, features}:
let cacheFeatures = cache.${packageId} or [];
combinedFeatures = sortedUnique (cacheFeatures ++ features);
in
if cache ? ${packageId} && cache.${packageId} == combinedFeatures
then cache
else mergePackageFeatures {
# This is purely for debugging.
dependencyPath = dependencyPath ++ [path crateConfigs.${packageId}.crateName];
features = combinedFeatures;
featuresByPackageId = cache;
inherit crateConfigs packageId;
});

cacheWithSelf =
let cacheFeatures = featuresByPackageId.${packageId} or [];
combinedFeatures = sortedUnique (cacheFeatures ++ expandedFeatures);
in featuresByPackageId // {
${packageId} = combinedFeatures;
};

cacheWithDependencies =
resolveDependencies cacheWithSelf "dep" (crateConfig.dependencies or []);
cacheWithAll =
resolveDependencies cacheWithDependencies "build" (crateConfig.buildDependencies or []);

in cacheWithAll;

/* Returns the enabled dependencies given the enabled features. */
filterEnabledDependencies = dependencies: features:
Expand All @@ -244,10 +253,8 @@ rec {
lib.filter
(dep:
let targetFunc = dep.target or (features: true);
target = targetFunc features;
in builtins.isString dep
|| target
&& (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep.name) features))
in targetFunc features
&& (!(dep.optional or false) || builtins.any (doesFeatureEnableDependency dep.name) features))
dependencies;

/* Returns whether the given feature should enable the given dependency. */
Expand Down Expand Up @@ -279,14 +286,13 @@ rec {
*/
dependencyFeatures = features: dependency:
assert (builtins.isList features);
assert (builtins.isAttrs dependency || builtins.isString dependency);
assert (builtins.isAttrs dependency);

let defaultOrNil = if builtins.isString dependency || dependency.usesDefaultFeatures or true
let defaultOrNil = if dependency.usesDefaultFeatures or true
then ["default"]
else [];
explicitFeatures = if builtins.isString dependency then [] else dependency.features or [];
explicitFeatures = dependency.features or [];
additionalDependencyFeatures =

let dependencyPrefix = dependency.name+"/";
dependencyFeatures =
builtins.filter (f: lib.hasPrefix dependencyPrefix f) features;
Expand Down
51 changes: 2 additions & 49 deletions crate2nix/templates/nix/crate2nix/tests/packageFeatures.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,111 +83,64 @@ let crateConfigs = {
};
};
};
packageFeatures = packageId: features: {
list = crate2nix.listOfPackageFeatures {inherit crateConfigs packageId features;};
merged = crate2nix.mergePackageFeatures {inherit crateConfigs packageId features;};
};
packageFeatures = packageId: features: crate2nix.mergePackageFeatures {inherit crateConfigs packageId features;};
in lib.runTests {

testNumDependencies = {
expr = packageFeatures "pkg_num" ["default"];
expected = {
list = [
{ packageId = "pkg_num"; features = ["default" "num-bigint/std" "std"]; }
{ packageId = "pkg_num_bigint"; features = ["std"]; }
];
merged = {
"pkg_num" = ["default" "num-bigint/std" "std"];
"pkg_num_bigint" = ["std"];
};
};
};

testNumTestDependencies = {
expr = packageFeatures "pkg_numtest" ["default"];
expected = {
list = [
{ packageId = "pkg_numtest"; features = ["default"]; }
{ packageId = "pkg_num"; features = ["default" "num-bigint/std" "std"]; }
{ packageId = "pkg_num_bigint"; features = ["std"]; }
];
merged = {
"pkg_numtest" = ["default"];
"pkg_num" = ["default" "num-bigint/std" "std"];
"pkg_num_bigint" = ["std"];
};
};
};

testTerminalPackageDependency = {
expr = packageFeatures "pkg_id1" [];
expected = {
list = [
{ packageId = "pkg_id1"; features = []; }
];
merged = {
"pkg_id1" = [];
};
};
};

testTerminalPackageDependencyWithDefault = {
expr = packageFeatures "pkg_id1" [ "default" ];
expected = {
list = [
{ packageId = "pkg_id1"; features = [ "default" ]; }
];
merged = {
"pkg_id1" = ["default"];
};
};
};

testRootPackage = {
expr = packageFeatures "pkg_root" [ "default" ];
expected = {
list = [
{ packageId = "pkg_root"; features = [ "default" ]; }
{ packageId = "pkg_id1"; features = [ "default" ]; }
{ packageId = "pkg_id3"; features = [ ]; }
];
merged = {
"pkg_root" = ["default"];
"pkg_id1" = ["default"];
"pkg_id3" = [];
};
};
};

testRootPackageWithOptional = {
expr = packageFeatures "pkg_root" [ "default" "optional_id2" ];
expected = {
list = [
{ packageId = "pkg_root"; features = [ "default" "optional_id2" ]; }
{ packageId = "pkg_id1"; features = [ "default" ]; }
{ packageId = "pkg_id2"; features = [ "default" ]; }
{ packageId = "pkg_id3"; features = [ ]; }
];
merged = {
"pkg_root" = ["default" "optional_id2"];
"pkg_id1" = ["default"];
"pkg_id2" = ["default"];
"pkg_id3" = [];
};
};
};

testPackageWithFeatureClash = {
expr = packageFeatures "pkg_with_feature_clash" [ ];
expected = {
list = [
{ packageId = "pkg_with_feature_clash"; features = []; }
{ packageId = "pkg_id1"; features = [ "default" ]; }
{ packageId = "pkg_id1"; features = [ "default" "for_build" ]; }
];
merged = {
"pkg_with_feature_clash" = [];
"pkg_id1" = [ "default" "for_build"];
};
};
};
}
Loading

0 comments on commit ec8aab5

Please sign in to comment.