From 51e054b01db7870bbb57e95f7e927f445bf928e4 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 11:29:17 -0500 Subject: [PATCH 01/18] refactor: preserve the full recipe layer list a bit longer --- .../detectables/bitbake/BitbakeExtractor.java | 2 +- .../BitbakeRecipesToLayerMapConverter.java | 12 ++++++------ .../bitbake/parse/BitbakeGraphTransformer.java | 14 ++++++++------ .../unit/BitbakeGraphTransformerTest.java | 17 +++++++++-------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index f054d0ae9e..7e81046390 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -76,7 +76,7 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List bitbakeRecipes = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); - Map recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes); + Map> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); CodeLocation codeLocation = new CodeLocation(dependencyGraph); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java index 71d7c4860b..64a9321e38 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java @@ -7,14 +7,14 @@ import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; public class BitbakeRecipesToLayerMapConverter { - public Map convert(List bitbakeRecipes) { - Map recipeNameToLayersMap = new HashMap<>(); - + // TODO: do we ever need a list? Could we just build a map in the first place? + public Map> convert(List bitbakeRecipes) { + Map> recipeNameToLayersMap = new HashMap<>(); for (BitbakeRecipe bitbakeRecipe : bitbakeRecipes) { - String key = bitbakeRecipe.getName(); - bitbakeRecipe.getLayerNames().stream().findFirst().ifPresent(layer -> recipeNameToLayersMap.put(key, layer)); + if (bitbakeRecipe.getLayerNames().size() > 0) { + recipeNameToLayersMap.put(bitbakeRecipe.getName(), bitbakeRecipe.getLayerNames()); + } } - return recipeNameToLayersMap; } } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java index f8d07aa92e..1251980713 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java @@ -1,6 +1,7 @@ package com.synopsys.integration.detectable.detectables.bitbake.parse; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -31,13 +32,13 @@ public BitbakeGraphTransformer(ExternalIdFactory externalIdFactory) { this.externalIdFactory = externalIdFactory; } - public DependencyGraph transform(BitbakeGraph bitbakeGraph, Map recipeLayerMap, Map imageRecipes, ExcludedDependencyTypeFilter excludedDependencyTypeFilter) { + public DependencyGraph transform(BitbakeGraph bitbakeGraph, Map> recipeLayerMap, Map imageRecipes, ExcludedDependencyTypeFilter excludedDependencyTypeFilter) { Map namesToExternalIds = generateExternalIds(bitbakeGraph, recipeLayerMap, imageRecipes, excludedDependencyTypeFilter); return buildGraph(bitbakeGraph, namesToExternalIds); } @NotNull - private Map generateExternalIds(final BitbakeGraph bitbakeGraph, final Map recipeLayerMap, final Map imageRecipes, ExcludedDependencyTypeFilter excludedDependencyTypeFilter) { + private Map generateExternalIds(final BitbakeGraph bitbakeGraph, final Map> recipeLayerMap, final Map imageRecipes, ExcludedDependencyTypeFilter excludedDependencyTypeFilter) { Map namesToExternalIds = new HashMap<>(); for (BitbakeNode bitbakeNode : bitbakeGraph.getNodes()) { String name = bitbakeNode.getName(); @@ -115,11 +116,12 @@ private String removeEpochPrefix(final String recipeVersion) { return epochlessRecipeVersion; } - private Optional generateExternalId(String dependencyName, String dependencyVersion, Map recipeLayerMap) { - String priorityLayerName = recipeLayerMap.get(dependencyName); + private Optional generateExternalId(String dependencyName, String dependencyVersion, Map> recipeLayerMap) { + List recipeLayerNames = recipeLayerMap.get(dependencyName); ExternalId externalId = null; - - if (priorityLayerName != null) { + if (recipeLayerNames != null) { + // TODO hoping we remove the reliance on this layer name and use task-depends.dot dependency layer name instead + String priorityLayerName = recipeLayerMap.get(dependencyName).get(0); externalId = externalIdFactory.createYoctoExternalId(priorityLayerName, dependencyName, dependencyVersion); } else { logger.debug("Failed to find component '{}' in component layer map.", dependencyName); diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java index 744200f445..5076c56b24 100644 --- a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java @@ -1,5 +1,6 @@ package com.synopsys.integration.detectable.detectables.bitbake.unit; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -29,9 +30,9 @@ public void parentHasChild() { bitbakeGraph.addNode("foobar", "12"); bitbakeGraph.addChild("example", "foobar"); - Map recipeToLayerMap = new HashMap<>(); - recipeToLayerMap.put("example", "meta"); - recipeToLayerMap.put("foobar", "meta"); + Map> recipeToLayerMap = new HashMap<>(); + recipeToLayerMap.put("example", Arrays.asList("meta")); + recipeToLayerMap.put("foobar", Arrays.asList("meta")); BitbakeGraphTransformer bitbakeGraphTransformer = new BitbakeGraphTransformer(new ExternalIdFactory()); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeToLayerMap, null, new ExcludedDependencyTypeFilter<>()); @@ -51,9 +52,9 @@ public void ignoredNoVersionRelationship() { bitbakeGraph.addNode("foobar", null); bitbakeGraph.addChild("example", "foobar"); - Map recipeToLayerMap = new HashMap<>(); - recipeToLayerMap.put("example", "meta"); - recipeToLayerMap.put("foobar", "meta"); + Map> recipeToLayerMap = new HashMap<>(); + recipeToLayerMap.put("example", Arrays.asList("meta")); + recipeToLayerMap.put("foobar", Arrays.asList("meta")); BitbakeGraphTransformer bitbakeGraphTransformer = new BitbakeGraphTransformer(new ExternalIdFactory()); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeToLayerMap, null, new ExcludedDependencyTypeFilter<>()); @@ -70,8 +71,8 @@ public void ignoredNoVersion() { BitbakeGraph bitbakeGraph = new BitbakeGraph(); bitbakeGraph.addNode("example", null); - Map recipeToLayerMap = new HashMap<>(); - recipeToLayerMap.put("example", "meta"); + Map> recipeToLayerMap = new HashMap<>(); + recipeToLayerMap.put("example", Arrays.asList("meta")); BitbakeGraphTransformer bitbakeGraphTransformer = new BitbakeGraphTransformer(new ExternalIdFactory()); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeToLayerMap, null, new ExcludedDependencyTypeFilter<>()); From 4eba072d5d9b12487f0a56b6018cfb111b51b3bf Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 11:48:47 -0500 Subject: [PATCH 02/18] feat: collect recipe layers only once --- .../detectables/bitbake/BitbakeExtractor.java | 46 +++++++++++-------- .../battery/detector/BitbakeBattery.java | 4 +- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 7e81046390..424b536465 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.NotImplementedException; @@ -63,29 +64,27 @@ public BitbakeExtractor(DetectableExecutableRunner executableRunner, GraphParser public Extraction extract(File sourceDirectory, File buildEnvScript, List sourceArguments, List packageNames, boolean followSymLinks, Integer searchDepth, ExcludedDependencyTypeFilter excludedDependencyTypeFilter, ExecutableTarget bash) { List codeLocations = new ArrayList<>(); - BitbakeSession bitbakeSession = new BitbakeSession(executableRunner, bitbakeRecipesParser, sourceDirectory, buildEnvScript, sourceArguments, bash, toolVersionLogger, buildFileFinder, bitbakeEnvironmentParser); bitbakeSession.logBitbakeVersion(); File buildDir = bitbakeSession.determineBuildDir(); BitbakeEnvironment bitbakeEnvironment = bitbakeSession.executeBitbakeForEnvironment(); - for (String packageName : packageNames) { - Map imageRecipes = null; - try { - if (excludedDependencyTypeFilter.shouldExcludeDependencyType(BitbakeDependencyType.BUILD)) { - imageRecipes = readImageRecipes(buildDir, packageName, bitbakeEnvironment, followSymLinks, searchDepth); + Optional> bitbakeRecipes = collectBitbakeRecipes(bitbakeSession); + if (bitbakeRecipes.isPresent()) { + for (String packageName : packageNames) { + Map imageRecipes = null; + try { + if (excludedDependencyTypeFilter.shouldExcludeDependencyType(BitbakeDependencyType.BUILD)) { + imageRecipes = readImageRecipes(buildDir, packageName, bitbakeEnvironment, followSymLinks, searchDepth); + } + BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, followSymLinks, searchDepth); + Map> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes.get()); + DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); + CodeLocation codeLocation = new CodeLocation(dependencyGraph); + codeLocations.add(codeLocation); + } catch (IOException | IntegrationException | ExecutableRunnerException | NotImplementedException | ExecutableFailedException e) { + logger.error(String.format("Failed to extract a Code Location while running Bitbake against package '%s': %s", packageName, e.getMessage())); + logger.debug(e.getMessage(), e); } - BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, followSymLinks, searchDepth); - List bitbakeRecipes = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); - Map> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes); - - DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); - CodeLocation codeLocation = new CodeLocation(dependencyGraph); - - codeLocations.add(codeLocation); - - } catch (IOException | IntegrationException | ExecutableRunnerException | NotImplementedException | ExecutableFailedException e) { - logger.error(String.format("Failed to extract a Code Location while running Bitbake against package '%s': %s", packageName, e.getMessage())); - logger.debug(e.getMessage(), e); } } @@ -105,6 +104,17 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> collectBitbakeRecipes(final BitbakeSession bitbakeSession) { + List bitbakeRecipes = null; + try { + bitbakeRecipes = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); + } catch (IOException | NotImplementedException | ExecutableFailedException e) { + logger.error(String.format("Error collecting recipe layers: %s", e.getMessage())); + logger.debug(e.getMessage(), e); + } + return Optional.ofNullable(bitbakeRecipes); + } + private Map readImageRecipes(File buildDir, String targetImageName, BitbakeEnvironment bitbakeEnvironment, boolean followSymLinks, int searchDepth) throws IntegrationException, IOException { File licenseManifestFile = buildFileFinder.findLicenseManifestFile(buildDir, targetImageName, bitbakeEnvironment, followSymLinks, searchDepth); logger.debug("Reading license.manifest file: {}", licenseManifestFile.getAbsolutePath()); diff --git a/src/test/java/com/synopsys/integration/detect/battery/detector/BitbakeBattery.java b/src/test/java/com/synopsys/integration/detect/battery/detector/BitbakeBattery.java index 1ea1f82642..f6211cff0c 100644 --- a/src/test/java/com/synopsys/integration/detect/battery/detector/BitbakeBattery.java +++ b/src/test/java/com/synopsys/integration/detect/battery/detector/BitbakeBattery.java @@ -14,7 +14,7 @@ void testIncludeAll() { DetectorBatteryTestRunner test = new DetectorBatteryTestRunner("bitbake-full", "bitbake/full"); test.sourceFileFromResource("oe-init-build-env"); test.sourceFileFromResource("task-depends.dot"); - test.executableFromResourceFiles(DetectProperties.DETECT_BASH_PATH.getProperty(), "pwd.xout", "environment.xout", "bitbake-g.xout", "bitbake-layers-show-recipes.xout"); + test.executableFromResourceFiles(DetectProperties.DETECT_BASH_PATH.getProperty(), "pwd.xout", "environment.xout", "bitbake-layers-show-recipes.xout", "bitbake-g.xout"); test.property("detect.bitbake.package.names", "core-image-sato"); test.expectBdioResources(); test.run(); @@ -26,7 +26,7 @@ void testExclBuild() { test.sourceFileFromResource("oe-init-build-env"); test.sourceFileFromResource("build/task-depends.dot"); test.sourceFileFromResource("build/tmp/deploy/licenses/core-image-sato-qemux86-64/license.manifest"); - test.executableFromResourceFiles(DetectProperties.DETECT_BASH_PATH.getProperty(), "pwd.xout", "environment.xout", "bitbake-g.xout", "bitbake-layers-show-recipes.xout"); + test.executableFromResourceFiles(DetectProperties.DETECT_BASH_PATH.getProperty(), "pwd.xout", "environment.xout", "bitbake-layers-show-recipes.xout", "bitbake-g.xout"); test.property("detect.bitbake.package.names", "core-image-sato"); test.property("detect.bitbake.dependency.types.excluded", "BUILD"); test.expectBdioResources(); From bfeb7ef85422f959b9e7ee75fc0fb988f9ffb95d Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 12:08:46 -0500 Subject: [PATCH 03/18] refactor: ShowRecipesResult object --- .../detectables/bitbake/BitbakeExtractor.java | 10 ++++---- .../detectables/bitbake/BitbakeSession.java | 2 +- .../bitbake/ShowRecipesResults.java | 24 +++++++++++++++++++ .../bitbake/parse/BitbakeRecipesParser.java | 11 +++++++-- 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 424b536465..3781e9ab3c 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -68,7 +68,8 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> bitbakeRecipes = collectBitbakeRecipes(bitbakeSession); + // TODO rename + Optional bitbakeRecipes = collectBitbakeRecipes(bitbakeSession); if (bitbakeRecipes.isPresent()) { for (String packageName : packageNames) { Map imageRecipes = null; @@ -77,7 +78,7 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes.get()); + Map> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes.get().getRecipes()); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); CodeLocation codeLocation = new CodeLocation(dependencyGraph); codeLocations.add(codeLocation); @@ -104,8 +105,9 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> collectBitbakeRecipes(final BitbakeSession bitbakeSession) { - List bitbakeRecipes = null; + private Optional collectBitbakeRecipes(final BitbakeSession bitbakeSession) { + // TODO rename + ShowRecipesResults bitbakeRecipes = null; try { bitbakeRecipes = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); } catch (IOException | NotImplementedException | ExecutableFailedException e) { diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java index 8a2ee01b1a..bbec77f697 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java @@ -96,7 +96,7 @@ public BitbakeEnvironment executeBitbakeForEnvironment() { } } - public List executeBitbakeForRecipeLayerCatalog() throws IOException, ExecutableFailedException { + public ShowRecipesResults executeBitbakeForRecipeLayerCatalog() throws IOException, ExecutableFailedException { ExecutableOutput executableOutput = runBitbake(BITBAKE_LAYERS_SHOW_RECIPES_COMMAND); return bitbakeRecipesParser.parseShowRecipes(executableOutput.getStandardOutputAsList()); } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java new file mode 100644 index 0000000000..4edab741ed --- /dev/null +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java @@ -0,0 +1,24 @@ +package com.synopsys.integration.detectable.detectables.bitbake; + +import java.util.List; +import java.util.Set; + +import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; + +public class ShowRecipesResults { + private final Set layerNames; + private final List recipes; + + public ShowRecipesResults(final Set layerNames, final List recipes) { + this.layerNames = layerNames; + this.recipes = recipes; + } + + public Set getLayerNames() { + return layerNames; + } + + public List getRecipes() { + return recipes; + } +} diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java index 3a30bee551..0d8214ca2e 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java @@ -1,11 +1,14 @@ package com.synopsys.integration.detectable.detectables.bitbake.parse; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.slf4j.LoggerFactory; +import com.synopsys.integration.detectable.detectables.bitbake.ShowRecipesResults; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; import com.synopsys.integration.log.IntLogger; import com.synopsys.integration.log.Slf4jIntLogger; @@ -17,8 +20,9 @@ public class BitbakeRecipesParser { * @param showRecipeLines is the executable output. * @return Recipe names mapped to a recipe's the layer names. */ - public List parseShowRecipes(List showRecipeLines) { + public ShowRecipesResults parseShowRecipes(List showRecipeLines) { List bitbakeRecipes = new ArrayList<>(); + Set layerNames = new HashSet<>(); boolean started = false; BitbakeRecipe currentRecipe = null; @@ -36,9 +40,12 @@ public List parseShowRecipes(List showRecipeLines) { if (currentRecipe != null) { bitbakeRecipes.add(currentRecipe); + if (currentRecipe.getLayerNames() != null) { + layerNames.addAll(currentRecipe.getLayerNames()); + } } - return bitbakeRecipes; + return new ShowRecipesResults(layerNames, bitbakeRecipes); } private BitbakeRecipe parseLine(String line, BitbakeRecipe currentRecipe, List bitbakeRecipes) { From 05ab2f72e7740cd886fb54aa2f6ad12694f95883 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 15:33:25 -0500 Subject: [PATCH 04/18] feat: parse dependency recipe layer from task-depends.dot --- .../detectables/bitbake/BitbakeExtractor.java | 7 ++- .../bitbake/model/BitbakeGraph.java | 6 +- .../bitbake/model/BitbakeNode.java | 9 +++ .../bitbake/parse/GraphParserTransformer.java | 55 +++++++++++++++++-- .../unit/BitbakeGraphTransformerTest.java | 10 ++-- .../unit/GraphParserTransformerTest.java | 22 +++++--- 6 files changed, 86 insertions(+), 23 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 3781e9ab3c..4a53f6f5e5 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.NotImplementedException; @@ -77,7 +78,7 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes.get().getRecipes()); DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); CodeLocation codeLocation = new CodeLocation(dependencyGraph); @@ -124,7 +125,7 @@ private Map readImageRecipes(File buildDir, String targetImageNa return licenseManifestParser.collectImageRecipes(licenseManifestLines); } - private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, File buildDir, String packageName, boolean followSymLinks, Integer searchDepth) + private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, File buildDir, String packageName, Set knownLayers, boolean followSymLinks, Integer searchDepth) throws ExecutableRunnerException, IOException, IntegrationException, ExecutableFailedException { File taskDependsFile = bitbakeSession.executeBitbakeForDependencies(buildDir, packageName, followSymLinks, searchDepth); if (logger.isTraceEnabled()) { @@ -132,6 +133,6 @@ private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, File bu } InputStream dependsFileInputStream = FileUtils.openInputStream(taskDependsFile); GraphParser graphParser = new GraphParser(dependsFileInputStream); - return graphParserTransformer.transform(graphParser); + return graphParserTransformer.transform(graphParser, knownLayers); } } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeGraph.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeGraph.java index d652e066f4..be01403ff4 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeGraph.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeGraph.java @@ -23,8 +23,10 @@ private BitbakeNode getOrCreate(String name) { return newNode; } - public void addNode(String name, @Nullable String version) { - getOrCreate(name).setVersion(version); + public void addNode(String name, @Nullable String version, @Nullable String layer) { + BitbakeNode node = getOrCreate(name); + node.setVersion(version); + node.setLayer(layer); } public void addChild(String parent, String child) { diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeNode.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeNode.java index d7f1b15213..7b314eaac8 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeNode.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/BitbakeNode.java @@ -7,6 +7,7 @@ public class BitbakeNode { private final String name; private String version = null; + private String layer = null; private final Set children = new HashSet<>(); public BitbakeNode(String name) {this.name = name;} @@ -19,6 +20,10 @@ public void setVersion(String version) { this.version = version; } + public void setLayer(String layer) { + this.layer = layer; + } + public String getName() { return name; } @@ -27,6 +32,10 @@ public Optional getVersion() { return Optional.ofNullable(version); } + public Optional getLayer() { + return Optional.ofNullable(layer); + } + public Set getChildren() { return children; } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java index a7c94db5b0..3159819e42 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java @@ -1,8 +1,11 @@ package com.synopsys.integration.detectable.detectables.bitbake.parse; +import java.util.List; import java.util.Optional; +import java.util.Set; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import com.paypal.digraph.parser.GraphEdge; import com.paypal.digraph.parser.GraphNode; @@ -10,14 +13,17 @@ import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeGraph; public class GraphParserTransformer { - public BitbakeGraph transform(GraphParser graphParser) { + public BitbakeGraph transform(GraphParser graphParser, Set layerNames) { BitbakeGraph bitbakeGraph = new BitbakeGraph(); for (GraphNode graphNode : graphParser.getNodes().values()) { String name = getNameFromNode(graphNode); - getVersionFromNode(graphNode).ifPresent( - version -> bitbakeGraph.addNode(name, version) - ); + Optional layer = getLayerFromNode(graphNode, layerNames); + // TODO refactor + Optional version = getVersionFromNode(graphNode); + if (version.isPresent()) { + bitbakeGraph.addNode(name, version.get(), layer.orElse(null)); + } } for (GraphEdge graphEdge : graphParser.getEdges().values()) { @@ -41,6 +47,26 @@ private Optional getVersionFromNode(GraphNode graphNode) { return attribute.map(this::getVersionFromLabel); } + private Optional getLayerFromNode(GraphNode graphNode, Set knownLayerNames) { + Optional labelAttribute = getLabelAttribute(graphNode); + // TODO refactor + if (labelAttribute.isPresent()) { + return getLayerFromLabel(labelAttribute.get(), knownLayerNames); + } else { + return Optional.empty(); + } + } + + private Optional getLabelFromNode(GraphNode graphNode, Set knownLayers) { + Optional attribute = getLabelAttribute(graphNode); + // TODO refactor + if (attribute.isPresent()) { + return getLayerFromLabel(attribute.get(), knownLayers); + } else { + return Optional.empty(); + } + } + private Optional getLabelAttribute(GraphNode graphNode) { String attribute = (String) graphNode.getAttribute("label"); Optional result = Optional.empty(); @@ -53,7 +79,26 @@ private Optional getLabelAttribute(GraphNode graphNode) { } private String getVersionFromLabel(String label) { - String[] components = label.split("\\\\n:|\\\\n"); + String[] components = getLabelParts(label); return components[1]; } + + private Optional getLayerFromLabel(String label, Set knownLayerNames) { + String[] components = getLabelParts(label); + if (components.length == 3) { + String bbPath = components[2]; + for (String candidateLayerName : knownLayerNames) { + String possibleLayerPathSubstring = "/" + candidateLayerName + "/"; + if (bbPath.contains(possibleLayerPathSubstring)) { + return Optional.of(candidateLayerName); + } + } + } + return Optional.empty(); + } + + @NotNull + private String[] getLabelParts(final String label) { + return label.split("\\\\n:|\\\\n"); + } } diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java index 5076c56b24..5c3ab4c576 100644 --- a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/BitbakeGraphTransformerTest.java @@ -26,8 +26,8 @@ public class BitbakeGraphTransformerTest { public void parentHasChild() { ExternalIdFactory externalIdFactory = new ExternalIdFactory(); BitbakeGraph bitbakeGraph = new BitbakeGraph(); - bitbakeGraph.addNode("example", "1:75-r50"); - bitbakeGraph.addNode("foobar", "12"); + bitbakeGraph.addNode("example", "1:75-r50", "meta"); + bitbakeGraph.addNode("foobar", "12", "meta"); bitbakeGraph.addChild("example", "foobar"); Map> recipeToLayerMap = new HashMap<>(); @@ -48,8 +48,8 @@ public void parentHasChild() { public void ignoredNoVersionRelationship() { ExternalIdFactory externalIdFactory = new ExternalIdFactory(); BitbakeGraph bitbakeGraph = new BitbakeGraph(); - bitbakeGraph.addNode("example", "75"); - bitbakeGraph.addNode("foobar", null); + bitbakeGraph.addNode("example", "75", "meta"); + bitbakeGraph.addNode("foobar", null, "meta"); bitbakeGraph.addChild("example", "foobar"); Map> recipeToLayerMap = new HashMap<>(); @@ -69,7 +69,7 @@ public void ignoredNoVersionRelationship() { public void ignoredNoVersion() { ExternalIdFactory externalIdFactory = new ExternalIdFactory(); BitbakeGraph bitbakeGraph = new BitbakeGraph(); - bitbakeGraph.addNode("example", null); + bitbakeGraph.addNode("example", null, "meta"); Map> recipeToLayerMap = new HashMap<>(); recipeToLayerMap.put("example", Arrays.asList("meta")); diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java index aa35c121c3..9b9ad4ba7c 100644 --- a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java @@ -1,6 +1,9 @@ package com.synopsys.integration.detectable.detectables.bitbake.unit; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -20,8 +23,9 @@ public void parsedVersionFromLabel() { HashMap edges = new HashMap<>(); HashMap nodes = new HashMap<>(); - addNode("name", "name\\n:version\\n/some/path/to.bb", nodes, edges); - BitbakeGraph bitbakeGraph = buildGraph(nodes, edges); + addNode("name", "name\\n:version\\n/some/meta/path/to.bb", nodes, edges); + Set knownLayers = new HashSet<>(Arrays.asList("aaa", "meta", "bbb")); + BitbakeGraph bitbakeGraph = buildGraph(nodes, edges, knownLayers); Assertions.assertEquals(1, bitbakeGraph.getNodes().size()); Assertions.assertEquals("version", bitbakeGraph.getNodes().get(0).getVersion().get()); @@ -32,10 +36,11 @@ public void parsedRelationship() { HashMap edges = new HashMap<>(); HashMap nodes = new HashMap<>(); - addNode("parent", "name\\n:parent.version\\n/some/path/to.bb", nodes, edges); - addNode("child", "name\\n:child.version\\n/some/path/to.bb", nodes, edges); + addNode("parent", "name\\n:parent.version\\n/some/meta/path/to.bb", nodes, edges); + addNode("child", "name\\n:child.version\\n/some/meta/path/to.bb", nodes, edges); addEdge("edge1", "parent", "child", nodes, edges); - BitbakeGraph bitbakeGraph = buildGraph(nodes, edges); + Set knownLayers = new HashSet<>(Arrays.asList("aaa", "meta", "bbb")); + BitbakeGraph bitbakeGraph = buildGraph(nodes, edges, knownLayers); Assertions.assertEquals(2, bitbakeGraph.getNodes().size()); Assertions.assertEquals(1, bitbakeGraph.getNodes().get(0).getChildren().size()); @@ -48,15 +53,16 @@ public void removedQuotesFromName() { HashMap nodes = new HashMap<>(); addNode("quotes\"removed", "example\\n:example\\n/example", nodes, edges); - BitbakeGraph bitbakeGraph = buildGraph(nodes, edges); + Set knownLayers = new HashSet<>(Arrays.asList("aaa", "meta", "bbb")); + BitbakeGraph bitbakeGraph = buildGraph(nodes, edges, knownLayers); Assertions.assertEquals(1, bitbakeGraph.getNodes().size()); Assertions.assertEquals("quotesremoved", bitbakeGraph.getNodes().get(0).getName()); } - private BitbakeGraph buildGraph(HashMap nodes, HashMap edges) { + private BitbakeGraph buildGraph(HashMap nodes, HashMap edges, Set knownLayers) { GraphParserTransformer graphParserTransformer = new GraphParserTransformer(); - BitbakeGraph bitbakeGraph = graphParserTransformer.transform(mockParser(nodes, edges)); + BitbakeGraph bitbakeGraph = graphParserTransformer.transform(mockParser(nodes, edges), knownLayers); return bitbakeGraph; } From ab538445f51bd4dc6a24bc525fe17e313a4be452 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 16:24:16 -0500 Subject: [PATCH 05/18] feat: prefer dependency recipe layer parsed from task-depends.dot --- .../parse/BitbakeGraphTransformer.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java index 1251980713..6530b3129a 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java @@ -6,6 +6,7 @@ import java.util.Optional; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +46,9 @@ private Map generateExternalIds(final BitbakeGraph bitbakeGr if (bitbakeNode.getVersion().isPresent()) { String version = bitbakeNode.getVersion().get(); + Optional actualLayer = bitbakeNode.getLayer(); if (excludedDependencyTypeFilter.shouldReportDependencyType(BitbakeDependencyType.BUILD) || !isBuildDependency(imageRecipes, name, version)) { - Optional dependency = generateExternalId(name, version, recipeLayerMap).map(Dependency::new); + Optional dependency = generateExternalId(name, version, actualLayer.orElse(null), recipeLayerMap).map(Dependency::new); dependency.ifPresent(value -> namesToExternalIds.put(bitbakeNode.getName(), value)); } } else if (name.startsWith(VIRTUAL_PREFIX)) { @@ -116,19 +118,33 @@ private String removeEpochPrefix(final String recipeVersion) { return epochlessRecipeVersion; } - private Optional generateExternalId(String dependencyName, String dependencyVersion, Map> recipeLayerMap) { + private Optional generateExternalId(String dependencyName, String dependencyVersion, @Nullable String dependencyLayer, Map> recipeLayerMap) { + // TODO sure feels like there is room for improvement in layer handling List recipeLayerNames = recipeLayerMap.get(dependencyName); ExternalId externalId = null; + // TODO does this test still make sense? if (recipeLayerNames != null) { + // TODO TEMP + if ((dependencyLayer != null) && !recipeLayerNames.contains(dependencyLayer)) { + logger.warn("recipe {} dependency layer name {} is not in recipe's layer list {}", dependencyName, dependencyLayer, recipeLayerNames); + } + if ((dependencyLayer != null) && !dependencyLayer.equals(recipeLayerNames.get(0))) { + logger.warn("recipe {} dependency layer name {} is not FIRST in recipe's layer list {}", dependencyName, dependencyLayer, recipeLayerNames); + } + if (dependencyLayer == null) { + logger.warn("Did not parse a layer for dependency {} from task-depends.dot; using {} instead", dependencyName, recipeLayerNames.get(0)); + dependencyLayer = recipeLayerNames.get(0); + } + ////////// // TODO hoping we remove the reliance on this layer name and use task-depends.dot dependency layer name instead - String priorityLayerName = recipeLayerMap.get(dependencyName).get(0); - externalId = externalIdFactory.createYoctoExternalId(priorityLayerName, dependencyName, dependencyVersion); + //String priorityLayerName = recipeLayerMap.get(dependencyName).get(0); + externalId = externalIdFactory.createYoctoExternalId(dependencyLayer, dependencyName, dependencyVersion); } else { logger.debug("Failed to find component '{}' in component layer map.", dependencyName); if (dependencyName.endsWith(NATIVE_SUFFIX)) { String alternativeName = dependencyName.replace(NATIVE_SUFFIX, ""); logger.debug("Generating alternative component name '{}' for '{}=={}'", alternativeName, dependencyName, dependencyVersion); - externalId = generateExternalId(alternativeName, dependencyVersion, recipeLayerMap).orElse(null); + externalId = generateExternalId(alternativeName, dependencyVersion, dependencyLayer, recipeLayerMap).orElse(null); } else { logger.debug("'{}=={}' is not an actual component. Excluding from graph.", dependencyName, dependencyVersion); } From e100f811205a49730bfaa38d6a551dc015b89376 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 16:35:37 -0500 Subject: [PATCH 06/18] style: added log msg --- .../detectables/bitbake/parse/BitbakeGraphTransformer.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java index 6530b3129a..c645e5c536 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java @@ -124,7 +124,7 @@ private Optional generateExternalId(String dependencyName, String de ExternalId externalId = null; // TODO does this test still make sense? if (recipeLayerNames != null) { - // TODO TEMP + // TODO May want to tone down some of this logging? Or at least refactor? if ((dependencyLayer != null) && !recipeLayerNames.contains(dependencyLayer)) { logger.warn("recipe {} dependency layer name {} is not in recipe's layer list {}", dependencyName, dependencyLayer, recipeLayerNames); } @@ -134,10 +134,9 @@ private Optional generateExternalId(String dependencyName, String de if (dependencyLayer == null) { logger.warn("Did not parse a layer for dependency {} from task-depends.dot; using {} instead", dependencyName, recipeLayerNames.get(0)); dependencyLayer = recipeLayerNames.get(0); + } else { + logger.trace("For dependency recipe {}: using layer {} parsed from task-depends.dot", dependencyName, dependencyLayer); } - ////////// - // TODO hoping we remove the reliance on this layer name and use task-depends.dot dependency layer name instead - //String priorityLayerName = recipeLayerMap.get(dependencyName).get(0); externalId = externalIdFactory.createYoctoExternalId(dependencyLayer, dependencyName, dependencyVersion); } else { logger.debug("Failed to find component '{}' in component layer map.", dependencyName); From 351d58ca672117e1397b59496724d391dfb84ed7 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 17:40:05 -0500 Subject: [PATCH 07/18] refactor: cleanup --- .../parse/BitbakeGraphTransformer.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java index c645e5c536..5fc044d930 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java @@ -119,33 +119,19 @@ private String removeEpochPrefix(final String recipeVersion) { } private Optional generateExternalId(String dependencyName, String dependencyVersion, @Nullable String dependencyLayer, Map> recipeLayerMap) { - // TODO sure feels like there is room for improvement in layer handling List recipeLayerNames = recipeLayerMap.get(dependencyName); ExternalId externalId = null; - // TODO does this test still make sense? if (recipeLayerNames != null) { - // TODO May want to tone down some of this logging? Or at least refactor? - if ((dependencyLayer != null) && !recipeLayerNames.contains(dependencyLayer)) { - logger.warn("recipe {} dependency layer name {} is not in recipe's layer list {}", dependencyName, dependencyLayer, recipeLayerNames); - } - if ((dependencyLayer != null) && !dependencyLayer.equals(recipeLayerNames.get(0))) { - logger.warn("recipe {} dependency layer name {} is not FIRST in recipe's layer list {}", dependencyName, dependencyLayer, recipeLayerNames); - } - if (dependencyLayer == null) { - logger.warn("Did not parse a layer for dependency {} from task-depends.dot; using {} instead", dependencyName, recipeLayerNames.get(0)); - dependencyLayer = recipeLayerNames.get(0); - } else { - logger.trace("For dependency recipe {}: using layer {} parsed from task-depends.dot", dependencyName, dependencyLayer); - } + dependencyLayer = chooseRecipeLayer(dependencyName, dependencyLayer, recipeLayerNames); externalId = externalIdFactory.createYoctoExternalId(dependencyLayer, dependencyName, dependencyVersion); } else { - logger.debug("Failed to find component '{}' in component layer map.", dependencyName); + logger.debug("Failed to find component '{}' in component layer map. [dependencyVersion: {}; dependencyLayer: {}", dependencyName, dependencyVersion, dependencyLayer); if (dependencyName.endsWith(NATIVE_SUFFIX)) { String alternativeName = dependencyName.replace(NATIVE_SUFFIX, ""); logger.debug("Generating alternative component name '{}' for '{}=={}'", alternativeName, dependencyName, dependencyVersion); externalId = generateExternalId(alternativeName, dependencyVersion, dependencyLayer, recipeLayerMap).orElse(null); } else { - logger.debug("'{}=={}' is not an actual component. Excluding from graph.", dependencyName, dependencyVersion); + logger.debug("'{}:{}' is not an actual component. Excluding from graph.", dependencyName, dependencyVersion); } } @@ -155,4 +141,14 @@ private Optional generateExternalId(String dependencyName, String de return Optional.ofNullable(externalId); } + + private String chooseRecipeLayer(final String dependencyName, @Nullable String dependencyLayer, final List recipeLayerNames) { + if (dependencyLayer == null) { + logger.debug("Did not parse a layer for dependency {} from task-depends.dot; falling back to layer {} (first from show-recipes output)", dependencyName, recipeLayerNames.get(0)); + dependencyLayer = recipeLayerNames.get(0); + } else { + logger.trace("For dependency recipe {}: using layer {} parsed from task-depends.dot", dependencyName, dependencyLayer); + } + return dependencyLayer; + } } From c4907058df1e3fbaf8e57687fe50cce036d11daa Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 18:01:20 -0500 Subject: [PATCH 08/18] refactor: when parsing show-layers, directly generate the map ultimately needed (avoiding a conversion to it later) --- .../detectables/bitbake/BitbakeExtractor.java | 8 ++------ .../BitbakeRecipesToLayerMapConverter.java | 20 ------------------- .../detectables/bitbake/BitbakeSession.java | 1 - .../bitbake/ShowRecipesResults.java | 13 ++++++------ .../bitbake/parse/BitbakeRecipesParser.java | 10 ++++++---- .../detectable/factory/DetectableFactory.java | 7 +------ 6 files changed, 15 insertions(+), 44 deletions(-) delete mode 100644 detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 4a53f6f5e5..7835748c77 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -25,7 +25,6 @@ import com.synopsys.integration.detectable.detectable.util.ExcludedDependencyTypeFilter; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeEnvironment; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeGraph; -import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeEnvironmentParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeGraphTransformer; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeRecipesParser; @@ -43,20 +42,18 @@ public class BitbakeExtractor { private final GraphParserTransformer graphParserTransformer; private final BitbakeGraphTransformer bitbakeGraphTransformer; private final BitbakeRecipesParser bitbakeRecipesParser; - private final BitbakeRecipesToLayerMapConverter bitbakeRecipesToLayerMap; private final ToolVersionLogger toolVersionLogger; private final BuildFileFinder buildFileFinder; private final LicenseManifestParser licenseManifestParser; private final BitbakeEnvironmentParser bitbakeEnvironmentParser; public BitbakeExtractor(DetectableExecutableRunner executableRunner, GraphParserTransformer graphParserTransformer, BitbakeGraphTransformer bitbakeGraphTransformer, - BitbakeRecipesParser bitbakeRecipesParser, BitbakeRecipesToLayerMapConverter bitbakeRecipesToLayerMap, ToolVersionLogger toolVersionLogger, BuildFileFinder buildFileFinder, + BitbakeRecipesParser bitbakeRecipesParser, ToolVersionLogger toolVersionLogger, BuildFileFinder buildFileFinder, LicenseManifestParser licenseManifestParser, BitbakeEnvironmentParser bitbakeEnvironmentParser) { this.executableRunner = executableRunner; this.graphParserTransformer = graphParserTransformer; this.bitbakeGraphTransformer = bitbakeGraphTransformer; this.bitbakeRecipesParser = bitbakeRecipesParser; - this.bitbakeRecipesToLayerMap = bitbakeRecipesToLayerMap; this.toolVersionLogger = toolVersionLogger; this.buildFileFinder = buildFileFinder; this.licenseManifestParser = licenseManifestParser; @@ -79,8 +76,7 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List> recipeNameToLayersMap = bitbakeRecipesToLayerMap.convert(bitbakeRecipes.get().getRecipes()); - DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, recipeNameToLayersMap, imageRecipes, excludedDependencyTypeFilter); + DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, bitbakeRecipes.get().getRecipesWithLayers(), imageRecipes, excludedDependencyTypeFilter); CodeLocation codeLocation = new CodeLocation(dependencyGraph); codeLocations.add(codeLocation); } catch (IOException | IntegrationException | ExecutableRunnerException | NotImplementedException | ExecutableFailedException e) { diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java deleted file mode 100644 index 64a9321e38..0000000000 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeRecipesToLayerMapConverter.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.synopsys.integration.detectable.detectables.bitbake; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; - -public class BitbakeRecipesToLayerMapConverter { - // TODO: do we ever need a list? Could we just build a map in the first place? - public Map> convert(List bitbakeRecipes) { - Map> recipeNameToLayersMap = new HashMap<>(); - for (BitbakeRecipe bitbakeRecipe : bitbakeRecipes) { - if (bitbakeRecipe.getLayerNames().size() > 0) { - recipeNameToLayersMap.put(bitbakeRecipe.getName(), bitbakeRecipe.getLayerNames()); - } - } - return recipeNameToLayersMap; - } -} diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java index bbec77f697..ecd9137f28 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeSession.java @@ -12,7 +12,6 @@ import com.synopsys.integration.detectable.detectable.executable.DetectableExecutableRunner; import com.synopsys.integration.detectable.detectable.executable.ExecutableFailedException; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeEnvironment; -import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeEnvironmentParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeRecipesParser; import com.synopsys.integration.detectable.util.ToolVersionLogger; diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java index 4edab741ed..dc77f60a5c 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/ShowRecipesResults.java @@ -1,24 +1,23 @@ package com.synopsys.integration.detectable.detectables.bitbake; import java.util.List; +import java.util.Map; import java.util.Set; -import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeRecipe; - public class ShowRecipesResults { private final Set layerNames; - private final List recipes; + private final Map> recipesWithLayers; - public ShowRecipesResults(final Set layerNames, final List recipes) { + public ShowRecipesResults(final Set layerNames, Map> recipesWithLayers) { this.layerNames = layerNames; - this.recipes = recipes; + this.recipesWithLayers = recipesWithLayers; } public Set getLayerNames() { return layerNames; } - public List getRecipes() { - return recipes; + public Map> getRecipesWithLayers() { + return recipesWithLayers; } } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java index 0d8214ca2e..19f10a02e2 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeRecipesParser.java @@ -1,8 +1,10 @@ package com.synopsys.integration.detectable.detectables.bitbake.parse; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.lang3.StringUtils; @@ -21,7 +23,7 @@ public class BitbakeRecipesParser { * @return Recipe names mapped to a recipe's the layer names. */ public ShowRecipesResults parseShowRecipes(List showRecipeLines) { - List bitbakeRecipes = new ArrayList<>(); + Map> bitbakeRecipes = new HashMap<>(); Set layerNames = new HashSet<>(); boolean started = false; @@ -39,7 +41,7 @@ public ShowRecipesResults parseShowRecipes(List showRecipeLines) { } if (currentRecipe != null) { - bitbakeRecipes.add(currentRecipe); + bitbakeRecipes.put(currentRecipe.getName(), currentRecipe.getLayerNames()); if (currentRecipe.getLayerNames() != null) { layerNames.addAll(currentRecipe.getLayerNames()); } @@ -48,11 +50,11 @@ public ShowRecipesResults parseShowRecipes(List showRecipeLines) { return new ShowRecipesResults(layerNames, bitbakeRecipes); } - private BitbakeRecipe parseLine(String line, BitbakeRecipe currentRecipe, List bitbakeRecipes) { + private BitbakeRecipe parseLine(String line, BitbakeRecipe currentRecipe, Map> bitbakeRecipes) { if (line.contains(":") && !line.startsWith(" ")) { // Parse beginning of new component if (currentRecipe != null) { - bitbakeRecipes.add(currentRecipe); + bitbakeRecipes.put(currentRecipe.getName(), currentRecipe.getLayerNames()); } String recipeName = line.replace(":", "").trim(); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java b/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java index 6de070ee6d..d2af881f38 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java @@ -47,7 +47,6 @@ import com.synopsys.integration.detectable.detectables.bitbake.BitbakeDetectable; import com.synopsys.integration.detectable.detectables.bitbake.BitbakeDetectableOptions; import com.synopsys.integration.detectable.detectables.bitbake.BitbakeExtractor; -import com.synopsys.integration.detectable.detectables.bitbake.BitbakeRecipesToLayerMapConverter; import com.synopsys.integration.detectable.detectables.bitbake.BuildFileFinder; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeEnvironmentParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeGraphTransformer; @@ -906,12 +905,8 @@ private BitbakeRecipesParser bitbakeRecipesParser() { return new BitbakeRecipesParser(); } - private BitbakeRecipesToLayerMapConverter bitbakeRecipesToLayerMap() { - return new BitbakeRecipesToLayerMapConverter(); - } - private BitbakeExtractor bitbakeExtractor() { - return new BitbakeExtractor(executableRunner, graphParserTransformer(), bitbakeGraphTransformer(), bitbakeRecipesParser(), bitbakeRecipesToLayerMap(), + return new BitbakeExtractor(executableRunner, graphParserTransformer(), bitbakeGraphTransformer(), bitbakeRecipesParser(), toolVersionLogger, new BuildFileFinder(fileFinder), new LicenseManifestParser(), new BitbakeEnvironmentParser()); } From e732b9e76e1853fd8c473f748ebaab33ada624a2 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 18:12:07 -0500 Subject: [PATCH 09/18] refactor: cleanup --- .../detectables/bitbake/BitbakeExtractor.java | 16 +++++++--------- .../bitbake/parse/GraphParserTransformer.java | 17 +---------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 7835748c77..b4020be6df 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -66,17 +66,16 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List bitbakeRecipes = collectBitbakeRecipes(bitbakeSession); - if (bitbakeRecipes.isPresent()) { + Optional showRecipesResults = collectBitbakeRecipes(bitbakeSession); + if (showRecipesResults.isPresent()) { for (String packageName : packageNames) { Map imageRecipes = null; try { if (excludedDependencyTypeFilter.shouldExcludeDependencyType(BitbakeDependencyType.BUILD)) { imageRecipes = readImageRecipes(buildDir, packageName, bitbakeEnvironment, followSymLinks, searchDepth); } - BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, bitbakeRecipes.get().getLayerNames(), followSymLinks, searchDepth); - DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, bitbakeRecipes.get().getRecipesWithLayers(), imageRecipes, excludedDependencyTypeFilter); + BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, showRecipesResults.get().getLayerNames(), followSymLinks, searchDepth); + DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, showRecipesResults.get().getRecipesWithLayers(), imageRecipes, excludedDependencyTypeFilter); CodeLocation codeLocation = new CodeLocation(dependencyGraph); codeLocations.add(codeLocation); } catch (IOException | IntegrationException | ExecutableRunnerException | NotImplementedException | ExecutableFailedException e) { @@ -103,15 +102,14 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List collectBitbakeRecipes(final BitbakeSession bitbakeSession) { - // TODO rename - ShowRecipesResults bitbakeRecipes = null; + ShowRecipesResults showRecipesResults = null; try { - bitbakeRecipes = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); + showRecipesResults = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); } catch (IOException | NotImplementedException | ExecutableFailedException e) { logger.error(String.format("Error collecting recipe layers: %s", e.getMessage())); logger.debug(e.getMessage(), e); } - return Optional.ofNullable(bitbakeRecipes); + return Optional.ofNullable(showRecipesResults); } private Map readImageRecipes(File buildDir, String targetImageName, BitbakeEnvironment bitbakeEnvironment, boolean followSymLinks, int searchDepth) throws IntegrationException, IOException { diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java index 3159819e42..de3ccaa25d 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java @@ -19,11 +19,7 @@ public BitbakeGraph transform(GraphParser graphParser, Set layerNames) { for (GraphNode graphNode : graphParser.getNodes().values()) { String name = getNameFromNode(graphNode); Optional layer = getLayerFromNode(graphNode, layerNames); - // TODO refactor - Optional version = getVersionFromNode(graphNode); - if (version.isPresent()) { - bitbakeGraph.addNode(name, version.get(), layer.orElse(null)); - } + getVersionFromNode(graphNode).ifPresent(ver -> bitbakeGraph.addNode(name, ver, layer.orElse(null))); } for (GraphEdge graphEdge : graphParser.getEdges().values()) { @@ -49,7 +45,6 @@ private Optional getVersionFromNode(GraphNode graphNode) { private Optional getLayerFromNode(GraphNode graphNode, Set knownLayerNames) { Optional labelAttribute = getLabelAttribute(graphNode); - // TODO refactor if (labelAttribute.isPresent()) { return getLayerFromLabel(labelAttribute.get(), knownLayerNames); } else { @@ -57,16 +52,6 @@ private Optional getLayerFromNode(GraphNode graphNode, Set known } } - private Optional getLabelFromNode(GraphNode graphNode, Set knownLayers) { - Optional attribute = getLabelAttribute(graphNode); - // TODO refactor - if (attribute.isPresent()) { - return getLayerFromLabel(attribute.get(), knownLayers); - } else { - return Optional.empty(); - } - } - private Optional getLabelAttribute(GraphNode graphNode) { String attribute = (String) graphNode.getAttribute("label"); Optional result = Optional.empty(); From b458c44da4cd21bdc682b2fe33078288fcdddce2 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 18 Jan 2022 18:20:52 -0500 Subject: [PATCH 10/18] refactor: cleanup --- .../detectable/detectables/bitbake/BitbakeExtractor.java | 5 ++--- .../detectable/detectables/bitbake/BuildFileFinder.java | 1 - .../detectables/bitbake/parse/GraphParserTransformer.java | 6 ++++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index b4020be6df..e0d67d0b0b 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -33,7 +33,6 @@ import com.synopsys.integration.detectable.extraction.Extraction; import com.synopsys.integration.detectable.util.ToolVersionLogger; import com.synopsys.integration.exception.IntegrationException; -import com.synopsys.integration.executable.ExecutableRunnerException; public class BitbakeExtractor { private final Logger logger = LoggerFactory.getLogger(this.getClass()); @@ -78,7 +77,7 @@ public Extraction extract(File sourceDirectory, File buildEnvScript, List readImageRecipes(File buildDir, String targetImageNa } private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, File buildDir, String packageName, Set knownLayers, boolean followSymLinks, Integer searchDepth) - throws ExecutableRunnerException, IOException, IntegrationException, ExecutableFailedException { + throws IOException, IntegrationException, ExecutableFailedException { File taskDependsFile = bitbakeSession.executeBitbakeForDependencies(buildDir, packageName, followSymLinks, searchDepth); if (logger.isTraceEnabled()) { logger.trace(FileUtils.readFileToString(taskDependsFile, Charset.defaultCharset())); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java index 5e81c50b72..f3cd91eafa 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java @@ -66,7 +66,6 @@ public File findLicenseManifestFile(File buildDir, String targetImageName, Bitba @NotNull private List generateListOfFiles(final File licensesDir) { - // TODO surely there's a single-line way to do this (via nio or apache FileUtils) File[] licensesDirContentsArray = licensesDir.listFiles(); if (licensesDirContentsArray == null) { return new ArrayList<>(0); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java index de3ccaa25d..b02657e25e 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java @@ -1,6 +1,5 @@ package com.synopsys.integration.detectable.detectables.bitbake.parse; -import java.util.List; import java.util.Optional; import java.util.Set; @@ -13,6 +12,9 @@ import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeGraph; public class GraphParserTransformer { + + public static final String LABEL_PATH_SEPARATOR = "/"; + public BitbakeGraph transform(GraphParser graphParser, Set layerNames) { BitbakeGraph bitbakeGraph = new BitbakeGraph(); @@ -73,7 +75,7 @@ private Optional getLayerFromLabel(String label, Set knownLayerN if (components.length == 3) { String bbPath = components[2]; for (String candidateLayerName : knownLayerNames) { - String possibleLayerPathSubstring = "/" + candidateLayerName + "/"; + String possibleLayerPathSubstring = LABEL_PATH_SEPARATOR + candidateLayerName + LABEL_PATH_SEPARATOR; if (bbPath.contains(possibleLayerPathSubstring)) { return Optional.of(candidateLayerName); } From 2cef78db178d0db5ea3ee24a7947ff351aca873e Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Thu, 20 Jan 2022 15:49:02 -0500 Subject: [PATCH 11/18] style: added a todo based on QA's experience with 7.9.0 reading an old/wrong task-depends.dot file --- .../detectable/detectables/bitbake/BuildFileFinder.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java index f3cd91eafa..7c68da8163 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java @@ -29,6 +29,9 @@ public BuildFileFinder(final FileFinder fileFinder) { } public File findTaskDependsFile(File sourceDir, File buildDir, boolean followSymLinks, Integer searchDepth) throws IntegrationException { + // TODO all this flexibility (depth, sourceDir) carries the risk of finding an old/wrong task-depends.dot file + // made worse by the fact that Detect leaves the task-depends.dot file behind after each target image / Detect run. + // How 'bout we don't leave them behind? Could we generate them in a temp dir? File taskDependsDotFile = fileFinder.findFile(buildDir, TASK_DEPENDS_FILE_NAME, followSymLinks, searchDepth); if (taskDependsDotFile == null) { logger.warn("Did not find {} in build dir {}; trying source dir", TASK_DEPENDS_FILE_NAME, buildDir.getAbsolutePath()); From c94d1a0973858aa320cbc1f031e8a3c0c690f034 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Mon, 24 Jan 2022 14:59:41 -0500 Subject: [PATCH 12/18] refactor: cleaning up the extractor a little bit --- .../detectables/bitbake/BitbakeExtractor.java | 54 +++++++++---------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index c13f27172b..a1839bc6ce 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -8,11 +8,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.NotImplementedException; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,27 +74,24 @@ public Extraction extract( bitbakeSession.logBitbakeVersion(); File buildDir = bitbakeSession.determineBuildDir(); BitbakeEnvironment bitbakeEnvironment = bitbakeSession.executeBitbakeForEnvironment(); - Optional showRecipesResults = collectBitbakeRecipes(bitbakeSession); - if (showRecipesResults.isPresent()) { - for (String packageName : packageNames) { - Map imageRecipes = null; - try { - if (dependencyTypeFilter.shouldExclude(BitbakeDependencyType.BUILD)) { - imageRecipes = readImageRecipes(buildDir, packageName, bitbakeEnvironment, followSymLinks, searchDepth); - } - BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, showRecipesResults.get().getLayerNames(), followSymLinks, searchDepth); - DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, showRecipesResults.get().getRecipesWithLayers(), imageRecipes); - CodeLocation codeLocation = new CodeLocation(dependencyGraph); - codeLocations.add(codeLocation); - } catch (IOException | IntegrationException | NotImplementedException | ExecutableFailedException e) { - logger.error(String.format("Failed to extract a Code Location while running Bitbake against package '%s': %s", packageName, e.getMessage())); - logger.debug(e.getMessage(), e); - } + ShowRecipesResults showRecipesResults; + try { + showRecipesResults = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); + } catch (IOException | ExecutableFailedException e) { + String msg = String.format("Error collecting recipe layer information from show-recipes command: %s", e.getMessage()); + logger.error(msg); + return new Extraction.Builder().failure(msg).build(); + } + for (String targetImage : packageNames) { + try { + CodeLocation codeLocation = generateCodeLocationForTargetImage(followSymLinks, searchDepth, dependencyTypeFilter, bitbakeSession, buildDir, bitbakeEnvironment, showRecipesResults, targetImage); + codeLocations.add(codeLocation); + } catch (IOException | IntegrationException | NotImplementedException | ExecutableFailedException e) { + logger.error(String.format("Failed to extract a Code Location while running Bitbake against package '%s': %s", targetImage, e.getMessage())); + logger.debug(e.getMessage(), e); } } - Extraction extraction; - if (codeLocations.isEmpty()) { extraction = new Extraction.Builder() .failure("No Code Locations were generated during extraction") @@ -105,19 +102,20 @@ public Extraction extract( .success(codeLocations) .build(); } - return extraction; } - private Optional collectBitbakeRecipes(final BitbakeSession bitbakeSession) { - ShowRecipesResults showRecipesResults = null; - try { - showRecipesResults = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); - } catch (IOException | NotImplementedException | ExecutableFailedException e) { - logger.error(String.format("Error collecting recipe layers: %s", e.getMessage())); - logger.debug(e.getMessage(), e); + @NotNull + private CodeLocation generateCodeLocationForTargetImage(final boolean followSymLinks, final Integer searchDepth, final EnumListFilter dependencyTypeFilter, final BitbakeSession bitbakeSession, final File buildDir, + final BitbakeEnvironment bitbakeEnvironment, final ShowRecipesResults showRecipesResults, final String packageName) throws IntegrationException, IOException, ExecutableFailedException { + Map imageRecipes = null; + if (dependencyTypeFilter.shouldExclude(BitbakeDependencyType.BUILD)) { + imageRecipes = readImageRecipes(buildDir, packageName, bitbakeEnvironment, followSymLinks, searchDepth); } - return Optional.ofNullable(showRecipesResults); + BitbakeGraph bitbakeGraph = generateBitbakeGraph(bitbakeSession, buildDir, packageName, showRecipesResults.getLayerNames(), followSymLinks, searchDepth); + DependencyGraph dependencyGraph = bitbakeGraphTransformer.transform(bitbakeGraph, showRecipesResults.getRecipesWithLayers(), imageRecipes); + CodeLocation codeLocation = new CodeLocation(dependencyGraph); + return codeLocation; } private Map readImageRecipes(File buildDir, String targetImageName, BitbakeEnvironment bitbakeEnvironment, boolean followSymLinks, int searchDepth) throws IntegrationException, IOException { From 073d430f397bb23bc59ae890ebfc6efcbeb110cd Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Mon, 24 Jan 2022 15:31:37 -0500 Subject: [PATCH 13/18] refactor: cleanup --- .../detectable/detectables/bitbake/BitbakeExtractor.java | 9 +++++++-- .../detectable/detectables/bitbake/BuildFileFinder.java | 3 --- .../bitbake/parse/BitbakeGraphTransformer.java | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index a1839bc6ce..104272efd1 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -125,8 +125,13 @@ private Map readImageRecipes(File buildDir, String targetImageNa return licenseManifestParser.collectImageRecipes(licenseManifestLines); } - private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, File buildDir, String packageName, Set knownLayers, boolean followSymLinks, Integer searchDepth) - throws IOException, IntegrationException, ExecutableFailedException { + private BitbakeGraph generateBitbakeGraph(BitbakeSession bitbakeSession, + File buildDir, + String packageName, + Set knownLayers, + boolean followSymLinks, + Integer searchDepth + ) throws IOException, IntegrationException, ExecutableFailedException { File taskDependsFile = bitbakeSession.executeBitbakeForDependencies(buildDir, packageName, followSymLinks, searchDepth); if (logger.isTraceEnabled()) { logger.trace(FileUtils.readFileToString(taskDependsFile, Charset.defaultCharset())); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java index 7c68da8163..f3cd91eafa 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BuildFileFinder.java @@ -29,9 +29,6 @@ public BuildFileFinder(final FileFinder fileFinder) { } public File findTaskDependsFile(File sourceDir, File buildDir, boolean followSymLinks, Integer searchDepth) throws IntegrationException { - // TODO all this flexibility (depth, sourceDir) carries the risk of finding an old/wrong task-depends.dot file - // made worse by the fact that Detect leaves the task-depends.dot file behind after each target image / Detect run. - // How 'bout we don't leave them behind? Could we generate them in a temp dir? File taskDependsDotFile = fileFinder.findFile(buildDir, TASK_DEPENDS_FILE_NAME, followSymLinks, searchDepth); if (taskDependsDotFile == null) { logger.warn("Did not find {} in build dir {}; trying source dir", TASK_DEPENDS_FILE_NAME, buildDir.getAbsolutePath()); diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java index 3fc7cdfcd5..5f92bb5a47 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/BitbakeGraphTransformer.java @@ -146,7 +146,7 @@ private Optional generateExternalId(String dependencyName, String de private String chooseRecipeLayer(final String dependencyName, @Nullable String dependencyLayer, final List recipeLayerNames) { if (dependencyLayer == null) { - logger.debug("Did not parse a layer for dependency {} from task-depends.dot; falling back to layer {} (first from show-recipes output)", dependencyName, recipeLayerNames.get(0)); + logger.warn("Did not parse a layer for dependency {} from task-depends.dot; falling back to layer {} (first from show-recipes output)", dependencyName, recipeLayerNames.get(0)); dependencyLayer = recipeLayerNames.get(0); } else { logger.trace("For dependency recipe {}: using layer {} parsed from task-depends.dot", dependencyName, dependencyLayer); From aae22f071254197e5229990409534050ae659bcd Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Mon, 24 Jan 2022 15:34:22 -0500 Subject: [PATCH 14/18] test: test layer parsing --- .../detectables/bitbake/unit/GraphParserTransformerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java index 9b9ad4ba7c..e939ece92e 100644 --- a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java @@ -29,6 +29,7 @@ public void parsedVersionFromLabel() { Assertions.assertEquals(1, bitbakeGraph.getNodes().size()); Assertions.assertEquals("version", bitbakeGraph.getNodes().get(0).getVersion().get()); + Assertions.assertEquals("meta", bitbakeGraph.getNodes().get(0).getLayer().get()); } @Test From d9eb3fae3ced41327cb7527261fee156ecfd4131 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 25 Jan 2022 16:46:03 -0500 Subject: [PATCH 15/18] refactor: minor simplification --- .../detectable/detectables/bitbake/BitbakeExtractor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 39910f67fe..69de48f964 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -86,8 +86,7 @@ public Extraction extract( } for (String targetImage : packageNames) { try { - CodeLocation codeLocation = generateCodeLocationForTargetImage(followSymLinks, searchDepth, dependencyTypeFilter, bitbakeSession, buildDir, bitbakeEnvironment, showRecipesResults, targetImage); - codeLocations.add(codeLocation); + codeLocations.add(generateCodeLocationForTargetImage(followSymLinks, searchDepth, dependencyTypeFilter, bitbakeSession, buildDir, bitbakeEnvironment, showRecipesResults, targetImage)); } catch (IOException | IntegrationException | NotImplementedException | ExecutableFailedException e) { logger.error(String.format("Failed to extract a Code Location while running Bitbake against package '%s': %s", targetImage, e.getMessage())); logger.debug(e.getMessage(), e); From e6fd2e562e8bab06c3733f4ac58ce7bbaf4e8488 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Tue, 25 Jan 2022 18:03:51 -0500 Subject: [PATCH 16/18] style(doc): bitbake fix doc and release note --- docs/markdown/packagemgrs/bitbake.md | 2 +- docs/markdown/releasenotes.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/markdown/packagemgrs/bitbake.md b/docs/markdown/packagemgrs/bitbake.md index 248634cbba..ca1ad217f7 100644 --- a/docs/markdown/packagemgrs/bitbake.md +++ b/docs/markdown/packagemgrs/bitbake.md @@ -17,10 +17,10 @@ script (by default: oe-init-build-env), and executes BitBake commands to collect The BitBake detector generates one codelocation for each given package (target image) name by performing the following steps: 1. Determines the build directory path by sourcing the given build environment setup script and determining the resulting working directory. 1. Runs 'bitbake --environment' to determine the currently-configured target machine architecture and licenses directory path. +1. Runs 'bitbake-layers show-recipes' to derive the list of layers and collect recipe layer information. 1. For each given package (target image) name: * If the user requested that build dependencies be excluded, [solution_name] locates and reads the license.manifest file for the given package (target image) and the currently-configured target machine architecture. This provides a list of recipes that are included in the target image (the non-build dependencies). * Runs 'bitbake -g {package}' to generate task-depends.dot, and reads recipes and dependency relationships from it. - * Runs 'bitbake-layers show-recipes' to derive each recipe's layer. The first layer listed for each recipe is used as the layer field of the KB externalID [solution_name] generates for the recipe. * If the user requested that build dependencies be excluded: [solution_name] excludes recipes not declared in license.manifest, as well as native recipes. [solution_name] always excludes virtual recipes (recipes with names prefixed with "virtual/"). * [solution_name] adds at the root level of the graph for the package (target image) each recipe found in task-depends.dot that is not excluded as described above. * Child (transitive) relationships are created from those root dependencies to their children (as specified in task-depends.dot). diff --git a/docs/markdown/releasenotes.md b/docs/markdown/releasenotes.md index ba5220d92a..f37e5aef6d 100644 --- a/docs/markdown/releasenotes.md +++ b/docs/markdown/releasenotes.md @@ -9,6 +9,9 @@ * The Go Mod Cli Detector no longer uses the "-u" flag when running **go list -m all**. This results in significantly faster scan times against Go Mod projects. * Deprecated the `detect.pnpm.dependency.types property` in favor of `detect.pnpm.dependency.types.excluded` for property consistency. +### Resolved issues +* (IDETECT-2925) Resolved an issue that could cause the Bitbake detector to incorrectly identify the layer of a dependency recipe. + ## Version 7.10.0 ### New features From 3f5c33e1b62f6ca256dc672aba9343971a544a19 Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Thu, 27 Jan 2022 19:37:03 -0500 Subject: [PATCH 17/18] chore: extractor throw, rather than catch, IO/Execute exceptions --- .../detectables/bitbake/BitbakeDetectable.java | 4 +++- .../detectables/bitbake/BitbakeExtractor.java | 11 ++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeDetectable.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeDetectable.java index bc21e56019..b745ae491d 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeDetectable.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeDetectable.java @@ -1,6 +1,7 @@ package com.synopsys.integration.detectable.detectables.bitbake; import java.io.File; +import java.io.IOException; import com.synopsys.integration.common.util.finder.FileFinder; import com.synopsys.integration.detectable.Detectable; @@ -9,6 +10,7 @@ import com.synopsys.integration.detectable.detectable.Requirements; import com.synopsys.integration.detectable.detectable.annotation.DetectableInfo; import com.synopsys.integration.detectable.detectable.exception.DetectableException; +import com.synopsys.integration.detectable.detectable.executable.ExecutableFailedException; import com.synopsys.integration.detectable.detectable.executable.resolver.BashResolver; import com.synopsys.integration.detectable.detectable.explanation.PropertyProvided; import com.synopsys.integration.detectable.detectable.result.DetectableResult; @@ -56,7 +58,7 @@ public DetectableResult extractable() throws DetectableException { } @Override - public Extraction extract(ExtractionEnvironment extractionEnvironment) { + public Extraction extract(ExtractionEnvironment extractionEnvironment) throws ExecutableFailedException, IOException { return bitbakeExtractor.extract( environment.getDirectory(), foundBuildEnvScript, diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java index 69de48f964..48e70c5e79 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/BitbakeExtractor.java @@ -70,20 +70,13 @@ public Extraction extract( Integer searchDepth, EnumListFilter dependencyTypeFilter, ExecutableTarget bash - ) { + ) throws ExecutableFailedException, IOException { List codeLocations = new ArrayList<>(); BitbakeSession bitbakeSession = new BitbakeSession(executableRunner, bitbakeRecipesParser, sourceDirectory, buildEnvScript, sourceArguments, bash, toolVersionLogger, buildFileFinder, bitbakeEnvironmentParser); bitbakeSession.logBitbakeVersion(); File buildDir = bitbakeSession.determineBuildDir(); BitbakeEnvironment bitbakeEnvironment = bitbakeSession.executeBitbakeForEnvironment(); - ShowRecipesResults showRecipesResults; - try { - showRecipesResults = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); - } catch (IOException | ExecutableFailedException e) { - String msg = String.format("Error collecting recipe layer information from show-recipes command: %s", e.getMessage()); - logger.error(msg); - return new Extraction.Builder().failure(msg).build(); - } + ShowRecipesResults showRecipesResults = bitbakeSession.executeBitbakeForRecipeLayerCatalog(); for (String targetImage : packageNames) { try { codeLocations.add(generateCodeLocationForTargetImage(followSymLinks, searchDepth, dependencyTypeFilter, bitbakeSession, buildDir, bitbakeEnvironment, showRecipesResults, targetImage)); From 94a213ed3c8de7d12e21d03391e82b5ee0365dba Mon Sep 17 00:00:00 2001 From: Steve Billings Date: Fri, 28 Jan 2022 11:56:33 -0500 Subject: [PATCH 18/18] refactor: added GraphNodeLabelParser and GraphNodeLabelDetails --- .../bitbake/model/GraphNodeLabelDetails.java | 25 +++++++ .../bitbake/parse/GraphNodeLabelParser.java | 44 +++++++++++++ .../bitbake/parse/GraphParserTransformer.java | 66 +++++++------------ .../detectable/factory/DetectableFactory.java | 3 +- .../unit/GraphNodeLabelParserTest.java | 39 +++++++++++ .../unit/GraphParserTransformerTest.java | 14 ++-- 6 files changed, 142 insertions(+), 49 deletions(-) create mode 100644 detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/GraphNodeLabelDetails.java create mode 100644 detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphNodeLabelParser.java create mode 100644 detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphNodeLabelParserTest.java diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/GraphNodeLabelDetails.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/GraphNodeLabelDetails.java new file mode 100644 index 0000000000..a38a85b6e0 --- /dev/null +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/model/GraphNodeLabelDetails.java @@ -0,0 +1,25 @@ +package com.synopsys.integration.detectable.detectables.bitbake.model; + +public class GraphNodeLabelDetails { + private final String nameType; + private final String version; + private final String recipeSpec; + + public GraphNodeLabelDetails(final String nameType, final String version, final String recipeSpec) { + this.nameType = nameType; + this.version = version; + this.recipeSpec = recipeSpec; + } + + public String getNameType() { + return nameType; + } + + public String getVersion() { + return version; + } + + public String getRecipeSpec() { + return recipeSpec; + } +} diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphNodeLabelParser.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphNodeLabelParser.java new file mode 100644 index 0000000000..2fdf92bb18 --- /dev/null +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphNodeLabelParser.java @@ -0,0 +1,44 @@ +package com.synopsys.integration.detectable.detectables.bitbake.parse; + +import java.util.Set; + +import org.jetbrains.annotations.NotNull; + +import com.synopsys.integration.detectable.detectables.bitbake.model.GraphNodeLabelDetails; +import com.synopsys.integration.exception.IntegrationException; + +// Example of a GraphNode label value: +// acl-native do_compile\n:2.3.1-r0\nvirtual:native:/workdir/poky/meta/recipes-support/attr/acl_2.3.1.bb +// Split into: +// acl-native do_compile -> GraphNodeLabelDetails.nameType +// 2.3.1-r0 -> GraphNodeLabelDetails.version +// virtual:native:/workdir/poky/meta/recipes-support/attr/acl_2.3.1.bb -> GraphNodeLabelDetails.recipeSpec +public class GraphNodeLabelParser { + private static final String LABEL_PATH_SEPARATOR = "/"; + + public String parseVersionFromLabel(String label) throws IntegrationException { + GraphNodeLabelDetails labelDetails = parseLabelParts(label); + return labelDetails.getVersion(); + } + + public String parseLayerFromLabel(String label, Set knownLayerNames) throws IntegrationException { + GraphNodeLabelDetails labelDetails = parseLabelParts(label); + String recipeSpec = labelDetails.getRecipeSpec(); + for (String candidateLayerName : knownLayerNames) { + String possibleLayerPathSubstring = LABEL_PATH_SEPARATOR + candidateLayerName + LABEL_PATH_SEPARATOR; + if (recipeSpec.contains(possibleLayerPathSubstring)) { + return candidateLayerName; + } + } + throw new IntegrationException(String.format("Graph Node recipe '%s' does not correspond to any known layer (%s)", label, knownLayerNames)); + } + + @NotNull + private GraphNodeLabelDetails parseLabelParts(final String label) throws IntegrationException { + String[] labelParts = label.split("\\\\n:|\\\\n"); + if (labelParts.length < 3) { + throw new IntegrationException(String.format("Error parsing Graph Node label '%s'; unexpected format.", label)); + } + return new GraphNodeLabelDetails(labelParts[0], labelParts[1], labelParts[2]); + } +} diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java index b02657e25e..b1a1ecb2f3 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/detectables/bitbake/parse/GraphParserTransformer.java @@ -4,29 +4,32 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; import com.paypal.digraph.parser.GraphEdge; import com.paypal.digraph.parser.GraphNode; import com.paypal.digraph.parser.GraphParser; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeGraph; +import com.synopsys.integration.exception.IntegrationException; public class GraphParserTransformer { + private final GraphNodeLabelParser graphNodeLabelParser; - public static final String LABEL_PATH_SEPARATOR = "/"; + public GraphParserTransformer(GraphNodeLabelParser graphNodeLabelParser) { + this.graphNodeLabelParser = graphNodeLabelParser; + } - public BitbakeGraph transform(GraphParser graphParser, Set layerNames) { + public BitbakeGraph transform(GraphParser graphParser, Set layerNames) throws IntegrationException { BitbakeGraph bitbakeGraph = new BitbakeGraph(); for (GraphNode graphNode : graphParser.getNodes().values()) { - String name = getNameFromNode(graphNode); - Optional layer = getLayerFromNode(graphNode, layerNames); - getVersionFromNode(graphNode).ifPresent(ver -> bitbakeGraph.addNode(name, ver, layer.orElse(null))); + String name = parseNameFromNode(graphNode); + Optional layer = parseLayerFromNode(graphNode, layerNames); + parseVersionFromNode(graphNode).ifPresent(ver -> bitbakeGraph.addNode(name, ver, layer.orElse(null))); } for (GraphEdge graphEdge : graphParser.getEdges().values()) { - String parent = getNameFromNode(graphEdge.getNode1()); - String child = getNameFromNode(graphEdge.getNode2()); + String parent = parseNameFromNode(graphEdge.getNode1()); + String child = parseNameFromNode(graphEdge.getNode2()); if (!parent.equals(child)) { bitbakeGraph.addChild(parent, child); } @@ -35,57 +38,36 @@ public BitbakeGraph transform(GraphParser graphParser, Set layerNames) { return bitbakeGraph; } - private String getNameFromNode(GraphNode graphNode) { + private String parseNameFromNode(GraphNode graphNode) { String[] nodeIdPieces = graphNode.getId().split(".do_"); return nodeIdPieces[0].replace("\"", ""); } - private Optional getVersionFromNode(GraphNode graphNode) { - Optional attribute = getLabelAttribute(graphNode); - return attribute.map(this::getVersionFromLabel); + private Optional parseVersionFromNode(GraphNode graphNode) throws IntegrationException { + Optional labelValue = getLabelAttribute(graphNode); + if (labelValue.isPresent()) { + return Optional.of(graphNodeLabelParser.parseVersionFromLabel(labelValue.get())); + } else { + return Optional.empty(); + } } - private Optional getLayerFromNode(GraphNode graphNode, Set knownLayerNames) { + private Optional parseLayerFromNode(GraphNode graphNode, Set knownLayerNames) throws IntegrationException { Optional labelAttribute = getLabelAttribute(graphNode); if (labelAttribute.isPresent()) { - return getLayerFromLabel(labelAttribute.get(), knownLayerNames); + return Optional.of(graphNodeLabelParser.parseLayerFromLabel(labelAttribute.get(), knownLayerNames)); } else { return Optional.empty(); } } private Optional getLabelAttribute(GraphNode graphNode) { - String attribute = (String) graphNode.getAttribute("label"); + String labelValue = (String) graphNode.getAttribute("label"); Optional result = Optional.empty(); - if (StringUtils.isNotBlank(attribute)) { - result = Optional.of(attribute); + if (StringUtils.isNotBlank(labelValue)) { + result = Optional.of(labelValue); } - return result; } - - private String getVersionFromLabel(String label) { - String[] components = getLabelParts(label); - return components[1]; - } - - private Optional getLayerFromLabel(String label, Set knownLayerNames) { - String[] components = getLabelParts(label); - if (components.length == 3) { - String bbPath = components[2]; - for (String candidateLayerName : knownLayerNames) { - String possibleLayerPathSubstring = LABEL_PATH_SEPARATOR + candidateLayerName + LABEL_PATH_SEPARATOR; - if (bbPath.contains(possibleLayerPathSubstring)) { - return Optional.of(candidateLayerName); - } - } - } - return Optional.empty(); - } - - @NotNull - private String[] getLabelParts(final String label) { - return label.split("\\\\n:|\\\\n"); - } } diff --git a/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java b/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java index 14cf1a0286..f7886ba1d5 100644 --- a/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java +++ b/detectable/src/main/java/com/synopsys/integration/detectable/factory/DetectableFactory.java @@ -51,6 +51,7 @@ import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeEnvironmentParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeGraphTransformer; import com.synopsys.integration.detectable.detectables.bitbake.parse.BitbakeRecipesParser; +import com.synopsys.integration.detectable.detectables.bitbake.parse.GraphNodeLabelParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.GraphParserTransformer; import com.synopsys.integration.detectable.detectables.bitbake.parse.LicenseManifestParser; import com.synopsys.integration.detectable.detectables.cargo.CargoDetectable; @@ -297,7 +298,7 @@ public BazelDetectable createBazelDetectable(DetectableEnvironment environment, public BitbakeDetectable createBitbakeDetectable(DetectableEnvironment environment, BitbakeDetectableOptions bitbakeDetectableOptions, BashResolver bashResolver) { BitbakeExtractor bitbakeExtractor = new BitbakeExtractor( executableRunner, - new GraphParserTransformer(), + new GraphParserTransformer(new GraphNodeLabelParser()), new BitbakeGraphTransformer(externalIdFactory, bitbakeDetectableOptions.getDependencyTypeFilter()), new BitbakeRecipesParser(), toolVersionLogger, diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphNodeLabelParserTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphNodeLabelParserTest.java new file mode 100644 index 0000000000..a01cb191cb --- /dev/null +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphNodeLabelParserTest.java @@ -0,0 +1,39 @@ +package com.synopsys.integration.detectable.detectables.bitbake.unit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import com.synopsys.integration.detectable.detectables.bitbake.parse.GraphNodeLabelParser; +import com.synopsys.integration.exception.IntegrationException; + +public class GraphNodeLabelParserTest { + + @Test + void testVersion() throws IntegrationException { + String labelValue = "acl-native do_compile\\n:2.3.1-r0\\nvirtual:native:/workdir/poky/meta/recipes-support/attr/acl_2.3.1.bb"; + GraphNodeLabelParser parser = new GraphNodeLabelParser(); + Set knownLayers = new HashSet<>(); + knownLayers.add("meta"); + + String version = parser.parseVersionFromLabel(labelValue); + + assertEquals("2.3.1-r0", version); + } + + @Test + void testLayer() throws IntegrationException { + String labelValue = "acl-native do_compile\\n:2.3.1-r0\\nvirtual:native:/workdir/poky/meta/recipes-support/attr/acl_2.3.1.bb"; + GraphNodeLabelParser parser = new GraphNodeLabelParser(); + Set knownLayers = new HashSet<>(); + knownLayers.add("meta"); + + String layer = parser.parseLayerFromLabel(labelValue, knownLayers); + + assertEquals("meta", layer); + } +} diff --git a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java index e939ece92e..dea6decc3e 100644 --- a/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java +++ b/detectable/src/test/java/com/synopsys/integration/detectable/detectables/bitbake/unit/GraphParserTransformerTest.java @@ -14,12 +14,14 @@ import com.paypal.digraph.parser.GraphParser; import com.synopsys.integration.detectable.annotations.UnitTest; import com.synopsys.integration.detectable.detectables.bitbake.model.BitbakeGraph; +import com.synopsys.integration.detectable.detectables.bitbake.parse.GraphNodeLabelParser; import com.synopsys.integration.detectable.detectables.bitbake.parse.GraphParserTransformer; +import com.synopsys.integration.exception.IntegrationException; @UnitTest public class GraphParserTransformerTest { @Test - public void parsedVersionFromLabel() { + public void parsedVersionFromLabel() throws IntegrationException { HashMap edges = new HashMap<>(); HashMap nodes = new HashMap<>(); @@ -33,7 +35,7 @@ public void parsedVersionFromLabel() { } @Test - public void parsedRelationship() { + public void parsedRelationship() throws IntegrationException { HashMap edges = new HashMap<>(); HashMap nodes = new HashMap<>(); @@ -49,11 +51,11 @@ public void parsedRelationship() { } @Test - public void removedQuotesFromName() { + public void removedQuotesFromName() throws IntegrationException { HashMap edges = new HashMap<>(); HashMap nodes = new HashMap<>(); - addNode("quotes\"removed", "example\\n:example\\n/example", nodes, edges); + addNode("quotes\"removed", "example\\n:example\\n/example/meta/some.bb", nodes, edges); Set knownLayers = new HashSet<>(Arrays.asList("aaa", "meta", "bbb")); BitbakeGraph bitbakeGraph = buildGraph(nodes, edges, knownLayers); @@ -61,8 +63,8 @@ public void removedQuotesFromName() { Assertions.assertEquals("quotesremoved", bitbakeGraph.getNodes().get(0).getName()); } - private BitbakeGraph buildGraph(HashMap nodes, HashMap edges, Set knownLayers) { - GraphParserTransformer graphParserTransformer = new GraphParserTransformer(); + private BitbakeGraph buildGraph(HashMap nodes, HashMap edges, Set knownLayers) throws IntegrationException { + GraphParserTransformer graphParserTransformer = new GraphParserTransformer(new GraphNodeLabelParser()); BitbakeGraph bitbakeGraph = graphParserTransformer.transform(mockParser(nodes, edges), knownLayers); return bitbakeGraph; }