From 29409258ed6dbc70a42cb1708abb208e9ee4bf8b Mon Sep 17 00:00:00 2001 From: Scott Patten Date: Thu, 25 Apr 2024 10:03:51 -0700 Subject: [PATCH] [ANE-1659] update cargo metadata ID parser (#1416) * add support for parsing cargo metadata IDs for cargo >= 1.77.0 * tests for parsing new version * clean up the comments * add a changelog entry * Mostly get the parsing right. * Make existing test cases work. * Cleanup, support a few additional spec formats. * Commentary. * Report version. * Handle an additional format correctly. Use final bit after a / for names when there isn't a better option. * Apply suggestions from code review Co-authored-by: Jeffrey Huynh * Address some final cleanup. --------- Co-authored-by: Christopher Sasarak Co-authored-by: Christopher Sasarak Co-authored-by: Jeffrey Huynh --- .github/workflows/build-all.yml | 2 + Changelog.md | 3 + src/Strategy/Cargo.hs | 129 +++++++++++++++++- test/Cargo/MetadataSpec.hs | 43 +++++- .../testdata/expected-metadata-1.77.2.json | 72 ++++++++++ test/Cargo/testdata/expected-metadata.json | 2 +- 6 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 test/Cargo/testdata/expected-metadata-1.77.2.json diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml index 6ecb11e832..7fed119755 100644 --- a/.github/workflows/build-all.yml +++ b/.github/workflows/build-all.yml @@ -198,6 +198,7 @@ jobs: run: | mkdir release find . -type f -path '*/fossa/fossa.exe' -exec cp {} release \; + ./release/fossa.exe --version cp target/release/diagnose.exe release cp target/release/millhone.exe release @@ -206,6 +207,7 @@ jobs: run: | mkdir release find . -type f -path '*/fossa/fossa' -exec cp {} release \; + ./release/fossa --version cp target/release/diagnose release cp target/release/millhone release diff --git a/Changelog.md b/Changelog.md index 5ca36910d0..194bedb5b6 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # FOSSA CLI Changelog +## v3.9.14 +- Update cargo strategy to parse new `cargo metadata` format for cargo >= 1.77.0 ([#1416](https://github.com/fossas/fossa-cli/pull/1416)). + ## v3.9.13 - Support GIT dependencies in Bundler projects ([#1403](https://github.com/fossas/fossa-cli/pull/1403/files)) - Reports: Increase the timeout when hitting the report generation API endpoint ([#1412](https://github.com/fossas/fossa-cli/pull/1412)). diff --git a/src/Strategy/Cargo.hs b/src/Strategy/Cargo.hs index 4fcb26b26e..86fca51224 100644 --- a/src/Strategy/Cargo.hs +++ b/src/Strategy/Cargo.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE OverloadedRecordDot #-} + module Strategy.Cargo ( discover, CargoMetadata (..), @@ -17,6 +19,7 @@ import App.Fossa.Analyze.LicenseAnalyze ( LicenseAnalyzeProject (licenseAnalyzeProject), ) import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject) +import Control.Applicative ((<|>)) import Control.Effect.Diagnostics ( Diagnostics, Has, @@ -30,18 +33,22 @@ import Control.Effect.Diagnostics ( import Control.Effect.Reader (Reader) import Data.Aeson.Types ( FromJSON (parseJSON), - Parser, ToJSON, withObject, (.:), (.:?), ) +import Data.Bifunctor (bimap, first) import Data.Foldable (for_, traverse_) +import Data.Functor (void) +import Data.List.NonEmpty qualified as NonEmpty import Data.Map.Strict qualified as Map -import Data.Maybe (catMaybes, isJust) +import Data.Maybe (catMaybes, fromMaybe, isJust) import Data.Set (Set) -import Data.String.Conversion (toText) +import Data.String.Conversion (toString, toText) +import Data.Text (Text) import Data.Text qualified as Text +import Data.Void (Void) import Diag.Diagnostic (renderDiagnostic) import Discovery.Filters (AllFilters) import Discovery.Simple (simpleDiscover) @@ -69,6 +76,18 @@ import Errata (Errata (..)) import GHC.Generics (Generic) import Graphing (Graphing, stripRoot) import Path (Abs, Dir, File, Path, parent, parseRelFile, toFilePath, ()) +import Text.Megaparsec ( + Parsec, + choice, + errorBundlePretty, + lookAhead, + optional, + parse, + takeRest, + takeWhile1P, + try, + ) +import Text.Megaparsec.Char (char, digitChar, space) import Toml (TomlCodec, dioptional, diwrap, (.=)) import Toml qualified import Types ( @@ -383,8 +402,104 @@ buildGraph meta = stripRoot $ traverse_ direct $ metadataWorkspaceMembers meta traverse_ addEdge $ resolvedNodes $ metadataResolve meta -parsePkgId :: Text.Text -> Parser PackageId -parsePkgId t = +-- | Custom Parsec type alias +type PkgSpecParser a = Parsec Void Text a + +-- | Parser for pre cargo v1.77.0 package ids. +oldPkgIdParser :: Text -> Either Text PackageId +oldPkgIdParser t = case Text.splitOn " " t of - [a, b, c] -> pure $ PackageId a b c - _ -> fail "malformed Package ID" + [a, b, c] -> Right $ PackageId a b c + _ -> Left $ "malformed Package ID: " <> t + +type PkgName = Text +type PkgVersion = Text + +parsePkgSpec :: PkgSpecParser PackageId +parsePkgSpec = eatSpaces (try longSpec <|> simplePkgSpec') + where + eatSpaces m = space *> m <* space + + -- Given the fragment: adler@1.0.2 + pkgName :: PkgSpecParser (PkgName, PkgVersion) + pkgName = do + -- Parse: adler + name <- takeWhile1P (Just "Package name") (`notElem` ['@', ':']) + -- Parse: @1.0.2 + version <- optional (choice [char '@', char ':'] *> semver) + -- It's possible to specify a name with no version, use "*" in this case. + pure (name, fromMaybe "*" version) + + simplePkgSpec' = + pkgName >>= \(name, version) -> + pure + PackageId + { pkgIdName = name + , pkgIdVersion = version + , pkgIdSource = "" + } + + -- Given the spec: registry+https://github.com/rust-lang/crates.io-index#adler@1.0.2 + longSpec :: PkgSpecParser PackageId + longSpec = do + -- Parse: registry+https + sourceInit <- takeWhile1P (Just "Initial URL") (/= ':') + -- Parse: ://github.com/rust-lang/crates.io-index + sourceRemaining <- takeWhile1P (Just "Remaining URL") (/= '#') + let pkgSource = sourceInit <> sourceRemaining + + -- In cases where we can't find a real name, use text after the last slash as a name. + -- e.g. file:///path/to/my/project/bar#2.0.0 has the name 'bar' + -- Cases of this are generally path dependencies. + let fallbackName = + maybe pkgSource NonEmpty.last + . NonEmpty.nonEmpty + . filter (/= "") + . Text.split (== '/') + $ sourceRemaining + + -- Parse (Optional): #adler@1.0.2 + nameVersion <- optional $ do + void $ char '#' + -- If there's only a version after '#', use the fallback as the name. + ((fallbackName,) <$> semver) + <|> pkgName + + let (name, version) = fromMaybe (fallbackName, "*") nameVersion + pure $ + PackageId + { pkgIdName = name + , pkgIdVersion = version + , pkgIdSource = pkgSource + } + + -- In the grammar, a semver always appears at the end of a string and is the only + -- non-terminal that starts with a digit, so don't bother parsing internally. + semver = try (lookAhead digitChar) *> takeRest + +-- Prior to Cargo 1.77.0, package IDs looked like this: +-- package version (source URL) +-- adler 1.0.2 (registry+https://github.com/rust-lang/crates.io-index) +-- +-- For 1.77.0 and later, they look like this: +-- registry source URL with a fragment of package@version +-- registry+https://github.com/rust-lang/crates.io-index#adler@1.0.2 +-- or +-- path source URL with a fragment of package@version +-- path+file:///Users/scott/projects/health-data/health_data#package_name@0.1.0 +-- or +-- path source URL with a fragment of version +-- In this case we grab the last entry in the path to use for the package name +-- path+file:///Users/scott/projects/health-data/health_data#0.1.0 +-- +-- Package Spec: https://doc.rust-lang.org/cargo/reference/pkgid-spec.html +parsePkgId :: MonadFail m => Text.Text -> m PackageId +parsePkgId t = either fail pure $ oldPkgIdParser' t <|> parseNewSpec + where + oldPkgIdParser' = first toString . oldPkgIdParser + + parseNewSpec :: Either String PackageId + parseNewSpec = + bimap errorBundlePretty (\p -> p{pkgIdSource = "(" <> p.pkgIdSource <> ")"}) + . parse parsePkgSpec "Cargo Package Spec" + $ t diff --git a/test/Cargo/MetadataSpec.hs b/test/Cargo/MetadataSpec.hs index 718cc89b8e..3a477c930d 100644 --- a/test/Cargo/MetadataSpec.hs +++ b/test/Cargo/MetadataSpec.hs @@ -14,8 +14,8 @@ import Graphing import Strategy.Cargo import Test.Hspec qualified as Test -expectedMetadata :: CargoMetadata -expectedMetadata = CargoMetadata [] [jfmtId] $ Resolve expectedResolveNodes +expectedMetadataPre1_77 :: CargoMetadata +expectedMetadataPre1_77 = CargoMetadata [] [jfmtId] $ Resolve expectedResolveNodes expectedResolveNodes :: [ResolveNode] expectedResolveNodes = [ansiTermNode, clapNode, jfmtNode] @@ -63,12 +63,47 @@ spec = do Test.it "should properly construct a resolution tree" $ case eitherDecode metaBytes of Left err -> Test.expectationFailure $ "failed to parse: " ++ err - Right result -> result `Test.shouldBe` expectedMetadata + Right result -> result `Test.shouldBe` expectedMetadataPre1_77 Test.describe "cargo metadata graph" $ do - let graph = pruneUnreachable $ buildGraph expectedMetadata + let graph = pruneUnreachable $ buildGraph expectedMetadataPre1_77 Test.it "should build the correct graph" $ do expectDeps [ansiTermDep, clapDep] graph expectEdges [(clapDep, ansiTermDep)] graph expectDirect [clapDep] graph + + post1_77MetadataParseSpec + +ansiTermIdNoVersion :: PackageId +ansiTermIdNoVersion = mkPkgId "ansi_term" "*" + +ansiTermNodeNoVersion :: ResolveNode +ansiTermNodeNoVersion = ResolveNode ansiTermIdNoVersion [] + +fooPathDepId :: PackageId +fooPathDepId = PackageId "foo" "*" "(file:///path/to/my/project/foo)" + +fooPathNode :: ResolveNode +fooPathNode = ResolveNode fooPathDepId [] + +barPathDepId :: PackageId +barPathDepId = PackageId "bar" "2.0.0" "(file:///path/to/my/project/bar)" + +barPathNode :: ResolveNode +barPathNode = ResolveNode barPathDepId [] + +expectedResolveNodesPost1_77 :: [ResolveNode] +expectedResolveNodesPost1_77 = [ansiTermNodeNoVersion, fooPathNode, barPathNode, clapNode, jfmtNode] + +expectedMetadataPost1_77 :: CargoMetadata +expectedMetadataPost1_77 = CargoMetadata [] [jfmtId] $ Resolve expectedResolveNodesPost1_77 + +post1_77MetadataParseSpec :: Test.Spec +post1_77MetadataParseSpec = + Test.describe "cargo metadata parser, >= 1.77.0" $ do + metaBytes <- Test.runIO $ BL.readFile "test/Cargo/testdata/expected-metadata-1.77.2.json" + Test.it "should properly construct a resolution tree" $ + case eitherDecode metaBytes of + Left err -> Test.expectationFailure $ "failed to parse: " ++ err + Right result -> result `Test.shouldBe` expectedMetadataPost1_77 diff --git a/test/Cargo/testdata/expected-metadata-1.77.2.json b/test/Cargo/testdata/expected-metadata-1.77.2.json new file mode 100644 index 0000000000..5d03a266b5 --- /dev/null +++ b/test/Cargo/testdata/expected-metadata-1.77.2.json @@ -0,0 +1,72 @@ +{ + "packages": [ ], + "workspace_members": [ + "path+file:///path/to/jfmt.rs#jfmt@1.0.0" + ], + "workspace_default_members": [ + "path+file:///path/to/jfmt.rs#jfmt@1.0.0" + ], + "resolve": { + "nodes": [ + { + "id": "registry+https://github.com/rust-lang/crates.io-index#ansi_term", + "dependencies": [ + "registry+https://github.com/rust-lang/crates.io-index#winapi@0.3.6" + ], + "deps": [], + "features": [] + }, + { + "id": "file:///path/to/my/project/foo", + "dependencies": [], + "deps": [], + "features": [] + }, + { + "id": "file:///path/to/my/project/bar#2.0.0", + "dependencies": [], + "deps": [], + "features": [] + }, + { + "id": "registry+https://github.com/rust-lang/crates.io-index#clap:2.33.0", + "deps": [ + { + "name": "ansi_term", + "pkg": "registry+https://github.com/rust-lang/crates.io-index#ansi_term@0.11.0", + "dep_kinds": [ + { + "kind": null, + "target": "cfg(not(windows))" + } + ] + } + ] + }, + { + "id": "path+file:///path/to/jfmt.rs#jfmt@1.0.0", + "dependencies": [ + "registry+https://github.com/rust-lang/crates.io-index#clap@2.33.0" + ], + "deps": [ + { + "name": "clap", + "pkg": "registry+https://github.com/rust-lang/crates.io-index#clap@2.33.0", + "dep_kinds": [ + { + "kind": null, + "target": null + } + ] + } + ], + "features": [] + } + ], + "root": "path+file:///path/to/jfmt.rs#jfmt@1.0.0" + }, + "target_directory": "/Users/scott/code/rust/jfmt.rs/target", + "version": 1, + "workspace_root": "/Users/scott/code/rust/jfmt.rs", + "metadata": null +} diff --git a/test/Cargo/testdata/expected-metadata.json b/test/Cargo/testdata/expected-metadata.json index 8ceea622f0..cc925f8806 100644 --- a/test/Cargo/testdata/expected-metadata.json +++ b/test/Cargo/testdata/expected-metadata.json @@ -39,4 +39,4 @@ } ] } -} \ No newline at end of file +}