Update changelog and retrieve changes from main

Changelog.md

@@ -1,8 +1,12 @@
 # FOSSA CLI Changelog
 
-## v3.8.30
+## Unreleased
 - Removes warnings and tracebacks to stderr [#1358](https://github.com/fossas/fossa-cli/pull/1358)
 
+## v3.8.30
+- Fix an issue with long-option syntax for older versions of `sbt` ([#1356](https://github.com/fossas/fossa-cli/pull/1356))
+- Debug: add more logging for debugging missing dependencies. ([#1360](https://github.com/fossas/fossa-cli/pull/1360))
+
 ## v3.8.29
 - Prevents showing SCM warnings in fossa analyze, test, and report [#1354](https://github.com/fossas/fossa-cli/pull/1354)
 - Pathfinder: Pathfinder has been deprecated and removed. ([#1350](https://github.com/fossas/fossa-cli/pull/1350))

docs/features/custom-license-and-keyword-searches.md

@@ -133,10 +133,10 @@ This will match "Proprietary License", "proprietary license", "proprietary Licen
 
 ### Ignoring case
 
-You can ignore case by using the case-insensitive [flag](./custom-license-and-keyword-search-regular-expression-syntax.md#character-classes#groupings-and-flags), `i`. This is done by adding `(i?)` to your regular expression. Everything after `(?i)` will be matched case-insenitively.
+You can ignore case by using the case-insensitive [flag](./custom-license-and-keyword-search-regular-expression-syntax.md#character-classes#groupings-and-flags), `i`. This is done by prepending `(?i)` to your regular expression. Everything after `(?i)` will be matched case-insenitively.
 
 ```
-(i?)custom license
+(?i)custom license
 ```
 
 This will match "Custom License", "CUSTOM LICENSE", "custom license" or "CusTOm LiCenSe".
@@ -248,7 +248,3 @@ You can also set the `ignoreOrgWideCustomLicenseScanConfigs` flag to `true` in y
 version: 3
 ignoreOrgWideCustomLicenseScanConfigs: true
 ```
-
-
-
-

docs/references/experimental/binary-discovery/README.md

@@ -1,6 +1,6 @@
 # Binary Discovery
 
-FOSSA supports the ability to flag all binary dependencies discovered in your project source tree as unlicensed dependencies via an opt-in flag.
+FOSSA supports the ability to flag all binary dependencies discovered in your project source tree as unlicensed dependencies via an opt-in flag (`--experimental-enable-binary-discovery`).
 
 The core idea behind this feature is that some organizations wish to validate all potential sources of intellectual property rights,
 and binaries are potential sources of intellectual property rights data for which we typically cannot automatically discover licensing information.

integration-test/Analysis/ScalaSpec.hs

@@ -4,12 +4,12 @@
 module Analysis.ScalaSpec (spec) where
 
 import Analysis.FixtureExpectationUtils (
-  DependencyResultsSummary (DependencyResultsSummary),
+  DependencyResultsSummary (..),
   testSuiteDepResultSummary,
  )
 import Analysis.FixtureUtils (
-  AnalysisTestFixture (AnalysisTestFixture),
-  FixtureArtifact (FixtureArtifact),
+  AnalysisTestFixture (..),
+  FixtureArtifact (..),
   FixtureEnvironment (NixEnv),
  )
 import Path (reldir)

src/App/Fossa/Analyze.hs

@@ -106,6 +106,7 @@ import Effect.Exec (Exec)
 import Effect.Logger (
   Logger,
   Severity (..),
+  logDebug,
   logInfo,
   logStdout,
  )
@@ -353,8 +354,10 @@ analyze cfg = Diag.context "fossa-analyze" $ do
           forkTask $ do
             res <- Diag.runDiagnosticsIO . diagToDebug . stickyLogStack . withEmptyStack $ Archive.discover (runAnalyzers filters) basedir ancestryDirect
             Diag.withResult SevError SevWarn res (const (pure ()))
+  logDebug $ "Unfiltered project scans: " <> pretty (show projectScans)
 
   let filteredProjects = mapMaybe toProjectResult projectScans
+  logDebug $ "Filtered project scans: " <> pretty (show filteredProjects)
 
   maybeEndpointAppVersion <- case destination of
     UploadScan apiOpts _ -> runFossaApiClient apiOpts $ do
@@ -377,6 +380,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
         $ traverse (enrichPathDependencies includeAll vendoredDepsOptions revision) filteredProjects
     (True, _) -> pure $ map enrichPathDependencies' filteredProjects
     (False, _) -> traverse (withPathDependencyNudge includeAll) filteredProjects
+  logDebug $ "Filtered projects with path dependencies: " <> pretty (show filteredProjects')
 
   let analysisResult = AnalysisScanResult projectScans vsiResults binarySearchResults manualSrcUnits dynamicLinkedResults maybeLernieResults
   renderScanSummary (severity cfg) maybeEndpointAppVersion analysisResult $ Config.filterSet cfg

src/Strategy/Gradle.hs

@@ -278,7 +278,9 @@ analyze foundTargets dir = withSystemTempDir "fossa-gradle" $ \tmpDir -> do
 
   let text = decodeUtf8 $ BL.toStrict stdout
   let resolvedProjects = ResolutionApi.parseResolutionApiJsonDeps text
+  logDebug $ "Resolved projects: " <> pretty (show resolvedProjects)
   let graphFromResolutionApi = ResolutionApi.buildGraph resolvedProjects (onlyConfigurations)
+  logDebug $ "Graph: " <> pretty (show graphFromResolutionApi)
 
   -- Log debug messages as seen in gradle script
   traverse_ (logDebug . pretty) (getDebugMessages text)

src/Strategy/Scala.hs

@@ -40,7 +40,6 @@ import Discovery.Walk (
   walkWithFilters',
  )
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   Has,
@@ -64,6 +63,7 @@ import Strategy.Maven.Pom qualified as Pom
 import Strategy.Maven.Pom.Closure (MavenProjectClosure, buildProjectClosures, closurePath)
 import Strategy.Maven.Pom.PomFile (RawPom (rawPomArtifact, rawPomGroup, rawPomVersion))
 import Strategy.Maven.Pom.Resolver (buildGlobalClosure)
+import Strategy.Scala.Common (mkSbtCommand)
 import Strategy.Scala.Errors (FailedToListProjects (FailedToListProjects), MaybeWithoutDependencyTreeTask (MaybeWithoutDependencyTreeTask), MissingFullDependencyPlugin (MissingFullDependencyPlugin))
 import Strategy.Scala.Plugin (genTreeJson, hasDependencyPlugins)
 import Strategy.Scala.SbtDependencyTree (SbtArtifact (SbtArtifact), analyze, sbtDepTreeCmd)
@@ -230,14 +230,7 @@ analyzeWithSbtDepTree (ScalaProject maybeDepTree _ closure) = context "Analyzing
       pure $ SbtArtifact groupId artifactId version
 
 makePomCmd :: Command
-makePomCmd =
-  Command
-    { cmdName = "sbt"
-    , -- --no-colors to disable ANSI escape codes
-      -- --batch to disable interactivity. normally, if an `sbt` command fails, it'll drop into repl mode: --batch will disable the repl.
-      cmdArgs = ["--no-colors", "--batch", "makePom"]
-    , cmdAllowErr = Never
-    }
+makePomCmd = mkSbtCommand "makePom"
 
 genPoms :: (Has Exec sig m, Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [MavenProjectClosure]
 genPoms projectDir = do

src/Strategy/Scala/Common.hs

@@ -1,11 +1,31 @@
 module Strategy.Scala.Common (
   removeLogPrefixes,
   SbtArtifact (..),
+  mkSbtCommand,
 ) where
 
 import Data.Maybe (fromMaybe)
 import Data.Text (Text)
 import Data.Text qualified as Text
+import Effect.Exec (AllowErr (Never), Command (..))
+
+-- | Generate an sbt sub-command using which turns off colored output and ensures it does not enter repl mode.
+--
+--  example:
+--  mkSbtCommand "dependencyTree"  results in "sbt -batch -no-colors dependencyTree" as a CLI command.
+mkSbtCommand :: Text -> Command
+mkSbtCommand cmdName =
+  Command
+    { cmdName = "sbt"
+    , -- Use single hyphens rather than double-hyphens for old sbt compatibility.
+      -- Ex: -batch instead of --batch
+      cmdArgs =
+        [ "-batch"
+        , "-no-colors"
+        , cmdName
+        ]
+    , cmdAllowErr = Never
+    }
 
 data SbtArtifact = SbtArtifact
   { groupId :: Text

src/Strategy/Scala/Plugin.hs

@@ -12,27 +12,18 @@ import Data.String.Conversion (ConvertUtf8 (decodeUtf8), toString)
 import Data.Text qualified as Text
 import Data.Text.Lazy qualified as TextLazy
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   Has,
   execThrow,
  )
 import Path (Abs, Dir, File, Path, mkRelFile, parent, parseAbsFile, (</>))
+import Strategy.Scala.Common (mkSbtCommand)
 
 -- | Returns list of plugins used by sbt.
 -- Ref: https://www.scala-sbt.org/1.x/docs/Plugins.html
 getPlugins :: Command
-getPlugins =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--no-colors" -- to disable ANSI escape codes
-        , "--batch" -- to disable interactivity
-        , "plugins"
-        ]
-    , cmdAllowErr = Never
-    }
+getPlugins = mkSbtCommand "plugins"
 
 -- | Returns list of plugins used by sbt.
 hasDependencyPlugins :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m (Bool, Bool)
@@ -54,17 +45,19 @@ hasDependencyPlugins projectDir = do
 --  ./tree.json
 --  ./tree.html
 --  ./tree.data.js
+--
+-- This command is documented as being invoked with capital "HTML" but older versions of sbt output an error like:
+--
+-- [error] Not a valid command: dependencyBrowseTreeHTML
+-- [error] Not a valid project ID: dependencyBrowseTreeHTML
+-- [error] Expected ':'
+-- [error] Not a valid key: dependencyBrowseTreeHTML (similar: dependencyBrowseTreeHtml, dependencyBrowseTree, dependencyBrowseTreeTarget)
+-- [error] dependencyBrowseTreeHTML
+--
+-- This command is only used when the plugin is installed explicitly, i.e. sbt < 1.4.
+-- Newer versions of sbt will use the built-in dependency graph plugin.
 mkDependencyBrowseTreeHTMLCmd :: Command
-mkDependencyBrowseTreeHTMLCmd =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--no-colors" -- to disable ANSI escape codes
-        , "--batch" -- to disable interactivity
-        , "dependencyBrowseTreeHTML"
-        ]
-    , cmdAllowErr = Never
-    }
+mkDependencyBrowseTreeHTMLCmd = mkSbtCommand "dependencyBrowseTreeHtml"
 
 genTreeJson :: (Has Exec sig m, Has Diagnostics sig m) => Path Abs Dir -> m [Path Abs File]
 genTreeJson projectDir = do

src/Strategy/Scala/SbtDependencyTree.hs

@@ -30,13 +30,12 @@ import DepTypes (
   VerConstraint (CEq),
  )
 import Effect.Exec (
-  AllowErr (Never),
   Command (..),
   Exec,
   ExecErr (CommandParseError),
  )
 import Graphing (Graphing, shrinkRoots, subGraphOf, unfold)
-import Strategy.Scala.Common (SbtArtifact (SbtArtifact, artifactId, groupId, version), removeLogPrefixes)
+import Strategy.Scala.Common (SbtArtifact (SbtArtifact, artifactId, groupId, version), mkSbtCommand, removeLogPrefixes)
 import Text.Megaparsec (
   MonadParsec (eof, takeWhileP, try),
   Parsec,
@@ -57,16 +56,7 @@ import Text.Megaparsec.Char.Lexer qualified as Lexer
 -- This only works with sbt v1.4.0 greater, or with sbt which has DependencyTreePlugin.
 -- Ref: https://www.scala-sbt.org/1.x/docs/sbt-1.4-Release-Notes.html#sbt-dependency-graph+is+in-sourced
 sbtDepTreeCmd :: Command
-sbtDepTreeCmd =
-  Command
-    { cmdName = "sbt"
-    , cmdArgs =
-        [ "--batch" -- ensure sbt does not enter repl mode!
-        , "--no-colors"
-        , "dependencyTree"
-        ]
-    , cmdAllowErr = Never
-    }
+sbtDepTreeCmd = mkSbtCommand "dependencyTree"
 
 data SbtDep = SbtDep
   { artifact :: SbtArtifact