Merge branch 'master' into ryanlink-patch-1

Changelog.md

@@ -1,5 +1,10 @@
 # FOSSA CLI Changelog
 
+## v3.8.30
+
+- Fix an issue with long-option syntax for older versions of `sbt` ([#1356](https://github.com/fossas/fossa-cli/pull/1356))
+- Debug: add more logging for debugging missing dependencies. ([#1360](https://github.com/fossas/fossa-cli/pull/1360))
+
 ## v3.8.29
 - Prevents showing SCM warnings in fossa analyze, test, and report [#1354](https://github.com/fossas/fossa-cli/pull/1354)
 - Pathfinder: Pathfinder has been deprecated and removed. ([#1350](https://github.com/fossas/fossa-cli/pull/1350))

integration-test/Analysis/ScalaSpec.hs

@@ -4,12 +4,12 @@
 module Analysis.ScalaSpec (spec) where
 
 import Analysis.FixtureExpectationUtils (
-  DependencyResultsSummary (DependencyResultsSummary),
+  DependencyResultsSummary (..),
   testSuiteDepResultSummary,
  )
 import Analysis.FixtureUtils (
-  AnalysisTestFixture (AnalysisTestFixture),
-  FixtureArtifact (FixtureArtifact),
+  AnalysisTestFixture (..),
+  FixtureArtifact (..),
   FixtureEnvironment (NixEnv),
  )
 import Path (reldir)

src/App/Fossa/Analyze.hs

@@ -106,6 +106,7 @@ import Effect.Exec (Exec)
 import Effect.Logger (
   Logger,
   Severity (..),
+  logDebug,
   logInfo,
   logStdout,
  )
@@ -353,8 +354,10 @@ analyze cfg = Diag.context "fossa-analyze" $ do
           forkTask $ do
             res <- Diag.runDiagnosticsIO . diagToDebug . stickyLogStack . withEmptyStack $ Archive.discover (runAnalyzers filters) basedir ancestryDirect
             Diag.withResult SevError SevWarn res (const (pure ()))
+  logDebug $ "Unfiltered project scans: " <> pretty (show projectScans)
 
   let filteredProjects = mapMaybe toProjectResult projectScans
+  logDebug $ "Filtered project scans: " <> pretty (show filteredProjects)
 
   maybeEndpointAppVersion <- case destination of
     UploadScan apiOpts _ -> runFossaApiClient apiOpts $ do
@@ -377,6 +380,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
         $ traverse (enrichPathDependencies includeAll vendoredDepsOptions revision) filteredProjects
     (True, _) -> pure $ map enrichPathDependencies' filteredProjects
     (False, _) -> traverse (withPathDependencyNudge includeAll) filteredProjects
+  logDebug $ "Filtered projects with path dependencies: " <> pretty (show filteredProjects')
 
   let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults manualSrcUnits dynamicLinkedResults maybeLernieResults
   renderScanSummary (severity cfg) maybeEndpointAppVersion analysisResult $ Config.filterSet cfg

src/Strategy/Gradle.hs

@@ -278,7 +278,9 @@ analyze foundTargets dir = withSystemTempDir "fossa-gradle" $ \tmpDir -> do
 
   let text = decodeUtf8 $ BL.toStrict stdout
   let resolvedProjects = ResolutionApi.parseResolutionApiJsonDeps text
+  logDebug $ "Resolved projects: " <> pretty (show resolvedProjects)
   let graphFromResolutionApi = ResolutionApi.buildGraph resolvedProjects (onlyConfigurations)
+  logDebug $ "Graph: " <> pretty (show graphFromResolutionApi)
 
   -- Log debug messages as seen in gradle script
   traverse_ (logDebug . pretty) (getDebugMessages text)

src/Strategy/Scala.hs

@@ -40,7 +40,6 @@ import Discovery.Walk (
   walkWithFilters',
  )
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   Has,
@@ -64,6 +63,7 @@ import Strategy.Maven.Pom qualified as Pom
 import Strategy.Maven.Pom.Closure (MavenProjectClosure, buildProjectClosures, closurePath)
 import Strategy.Maven.Pom.PomFile (RawPom (rawPomArtifact, rawPomGroup, rawPomVersion))
 import Strategy.Maven.Pom.Resolver (buildGlobalClosure)
+import Strategy.Scala.Common (mkSbtCommand)
 import Strategy.Scala.Errors (FailedToListProjects (FailedToListProjects), MaybeWithoutDependencyTreeTask (MaybeWithoutDependencyTreeTask), MissingFullDependencyPlugin (MissingFullDependencyPlugin))
 import Strategy.Scala.Plugin (genTreeJson, hasDependencyPlugins)
 import Strategy.Scala.SbtDependencyTree (SbtArtifact (SbtArtifact), analyze, sbtDepTreeCmd)
@@ -230,14 +230,7 @@ analyzeWithSbtDepTree (ScalaProject maybeDepTree _ closure) = context "Analyzing
       pure $ SbtArtifact groupId artifactId version
 
 makePomCmd :: Command
-makePomCmd =
-  Command
-    { cmdName = "sbt"
-    , -- --no-colors to disable ANSI escape codes
-      -- --batch to disable interactivity. normally, if an `sbt` command fails, it'll drop into repl mode: --batch will disable the repl.
-      cmdArgs = ["--no-colors", "--batch", "makePom"]
-    , cmdAllowErr = Never
-    }
+makePomCmd = mkSbtCommand "makePom"
 
 genPoms :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [MavenProjectClosure]
 genPoms projectDir = do

src/Strategy/Scala/Common.hs

@@ -1,11 +1,31 @@
 module Strategy.Scala.Common (
   removeLogPrefixes,
   SbtArtifact (..),
+  mkSbtCommand,
 ) where
 
 import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text qualified as Text
+import Effect.Exec (AllowErr (Never), Command (..))
+
+-- | Generate an sbt sub-command using which turns off colored output and ensures it does not enter repl mode.
+--
+--  example:
+--  mkSbtCommand "dependencyTree"  results in "sbt -batch -no-colors dependencyTree" as a CLI command.
+mkSbtCommand :: Text -> Command
+mkSbtCommand cmdName =
+  Command
+    { cmdName = "sbt"
+    , -- Use single hyphens rather than double-hyphens for old sbt compatibility.
+      -- Ex: -batch instead of --batch
+      cmdArgs =
+        [ "-batch"
+        , "-no-colors"
+        , cmdName
+        ]
+    , cmdAllowErr = Never
+    }
 
 data SbtArtifact = SbtArtifact
   { groupId :: Text

src/Strategy/Scala/Plugin.hs

@@ -12,27 +12,18 @@ import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString)
 import Data.Text qualified as Text
 import Data.Text.Lazy qualified as TextLazy
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   Has,
   execThrow,
  )
 import Path (Abs, Dir, File, Path, mkRelFile, parent, parseAbsFile, (</>))
+import Strategy.Scala.Common (mkSbtCommand)
 
 -- | Returns list of plugins used by sbt.
 -- Ref: https://www.scala-sbt.org/1.x/docs/Plugins.html
 getPlugins :: Command
-getPlugins =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--no-colors" -- to disable ANSI escape codes
-        , "--batch" -- to disable interactivity
-        , "plugins"
-        ]
-    , cmdAllowErr = Never
-    }
+getPlugins = mkSbtCommand "plugins"
 
 -- | Returns list of plugins used by sbt.
 hasDependencyPlugins :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m (Bool, Bool)
@@ -54,17 +45,19 @@ hasDependencyPlugins projectDir = do
 --  ./tree.json
 --  ./tree.html
 --  ./tree.data.js
+--
+-- This command is documented as being invoked with capital "HTML" but older versions of sbt output an error like:
+--
+-- [error] Not a valid command: dependencyBrowseTreeHTML
+-- [error] Not a valid project ID: dependencyBrowseTreeHTML
+-- [error] Expected ':'
+-- [error] Not a valid key: dependencyBrowseTreeHTML (similar: dependencyBrowseTreeHtml, dependencyBrowseTree, dependencyBrowseTreeTarget)
+-- [error] dependencyBrowseTreeHTML
+--
+-- This command is only used when the plugin is installed explicitly, i.e. sbt < 1.4.
+-- Newer versions of sbt will use the built-in dependency graph plugin.
 mkDependencyBrowseTreeHTMLCmd :: Command
-mkDependencyBrowseTreeHTMLCmd =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--no-colors" -- to disable ANSI escape codes
-        , "--batch" -- to disable interactivity
-        , "dependencyBrowseTreeHTML"
-        ]
-    , cmdAllowErr = Never
-    }
+mkDependencyBrowseTreeHTMLCmd = mkSbtCommand "dependencyBrowseTreeHtml"
 
 genTreeJson :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m [Path Abs File]
 genTreeJson projectDir = do

src/Strategy/Scala/SbtDependencyTree.hs

@@ -30,13 +30,12 @@ import DepTypes (
   VerConstraint (CEq),
  )
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   ExecErr (CommandParseError),
  )
 import Graphing (Graphing, shrinkRoots, subGraphOf, unfold)
-import Strategy.Scala.Common (SbtArtifact (SbtArtifact, artifactId, groupId, version), removeLogPrefixes)
+import Strategy.Scala.Common (SbtArtifact (SbtArtifact, artifactId, groupId, version), mkSbtCommand, removeLogPrefixes)
 import Text.Megaparsec (
   MonadParsec (eof, takeWhileP, try),
   Parsec,
@@ -57,16 +56,7 @@ import Text.Megaparsec.Char.Lexer qualified as Lexer
 -- This only works with sbt v1.4.0 greater, or with sbt which has DependencyTreePlugin.
 -- Ref: https://www.scala-sbt.org/1.x/docs/sbt-1.4-Release-Notes.html#sbt-dependency-graph+is+in-sourced
 sbtDepTreeCmd :: Command
-sbtDepTreeCmd =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--batch" -- ensure sbt does not enter repl mode!
-        , "--no-colors"
-        , "dependencyTree"
-        ]
-    , cmdAllowErr = Never
-    }
+sbtDepTreeCmd = mkSbtCommand "dependencyTree"
 
 data SbtDep = SbtDep
   { artifact :: SbtArtifact