Skip to content

Commit

Permalink
saving wip
Browse files Browse the repository at this point in the history
  • Loading branch information
meghfossa committed Apr 30, 2024
1 parent 5406f32 commit 7287894
Show file tree
Hide file tree
Showing 11 changed files with 1,134 additions and 103 deletions.
3 changes: 2 additions & 1 deletion spectrometer.cabal
Expand Up @@ -161,7 +161,7 @@ common deps
, yarn-lock ^>=0.6.5
, zip ^>=2.0.0
, zlib ^>=0.6.2.1

, validation-selective >= 0.1.0 && < 0.3
if !os(windows)
-- included with GHC, consider upgrading this as part of a ghc upgrade
build-depends: unix ^>=2.7.2.2
Expand Down Expand Up @@ -259,6 +259,7 @@ library
App.Fossa.Report
App.Fossa.Report.Attribution
App.Fossa.RunThemis
Data.Toml.Extra
App.Fossa.Snippets
App.Fossa.Snippets.Analyze
App.Fossa.Snippets.Commit
Expand Down
201 changes: 201 additions & 0 deletions src/Data/Toml/Extra.hs
@@ -0,0 +1,201 @@
{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
{-# HLINT ignore "Redundant lambda" #-}
{-# HLINT ignore "Use for_" #-}
module Data.Toml.Extra (
tableMap'
) where

import Toml.Codec.BiMap (BiMap (..), TomlBiMap)
import Toml.Codec.Code (execTomlCodec)

Check failure on line 9 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

The import of ‘Toml.Codec.Code’ is redundant
import Toml.Codec.Combinator.Common (whenLeftBiMapError)
import Toml.Codec.Types (Codec (..), TomlCodec, TomlEnv, TomlState (..))
import Toml.Type.Key
import Toml.Type.TOML (TOML (..), insertTable, insertTableArrays)

Check failure on line 13 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

The import of ‘insertTableArrays’
import qualified Toml.Type.PrefixTree as Prefix
import Control.Applicative (empty)
import Control.Monad (forM_)
import Control.Monad.State (gets, modify)
import qualified Data.Map.Strict as Map
import Data.Maybe (fromMaybe)
import Data.Traversable (for)
import Data.HashMap.Strict (HashMap)

Check failure on line 21 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

The import of ‘Data.HashMap.Strict’ is redundant
import Data.Hashable (Hashable)

Check failure on line 22 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

The import of ‘Data.Hashable’ is redundant
import qualified Data.HashMap.Strict as HashMap
import Validation (Validation (..))
import Data.Map.Strict (Map)
import Prelude hiding (lookup)
import Debug.Trace (trace)
import qualified Data.List.NonEmpty as NE
import Toml.Codec (TomlDecodeError)
-- import qualified Strategy.Python.Poetry.PyProject as group.group.dev.dev

-- tables ::

applicableKey :: Show a => Key -> Prefix.PrefixMap a -> [Key]
applicableKey key prefMap = map simplify $ filter (compareKey' key) allKeys
where
listed = trace ("input.simplify.listed': " ++ show (Prefix.toList prefMap)) $ Prefix.toList prefMap

allKeys :: [Key]
allKeys = trace ("input.simplify.allKeys': " ++ show (map (simplify . fst) $ Prefix.toList prefMap)) $ map (simplify . fst) $ listed

simplify :: Key -> Key
simplify key' = trace ("input.simplify': " ++ show key') $ Key . removeConsecutiveDuplicates . unKey $ key'

removeConsecutiveDuplicates :: Eq a => NE.NonEmpty a -> NE.NonEmpty a
removeConsecutiveDuplicates (x NE.:| []) = x NE.:| []
removeConsecutiveDuplicates (x NE.:| (y:ys))
| x == y = removeConsecutiveDuplicates (x NE.:| ys)
| otherwise = case NE.nonEmpty (y:ys) of
Just ne -> NE.singleton x <> (removeConsecutiveDuplicates ne)
_ -> x NE.:| []

tableMap'
:: forall k v
. (Ord k, Show k, Show v)
=> TomlBiMap Key k
-- ^ Bidirectional converter between TOML 'Key's and 'Map' keys
-> (Key -> TomlCodec v)
-- ^ Codec for 'Map' values for the corresponding 'Key'
-> Key
-- ^ Table name for 'Map'
-> TomlCodec (Map k v)
tableMap' = internalTableMap Map.empty Map.toList Map.fromList Map.unions

-- type TomlEnv a = TOML -> Validation [TomlDecodeError] a

internalTableMap
:: forall map k v
. Show map => map -- ^ empty map

Check failure on line 69 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

Redundant constraint: Show map
-> (map -> [(k, v)]) -- ^ toList function
-> ([(k, v)] -> map) -- ^ fromList function
-> ([map] -> map) -- ^ union function
-> TomlBiMap Key k
-- ^ Bidirectional converter between TOML 'Key's and Map keys
-> (Key -> TomlCodec v)
-- ^ Codec for Map values for the corresponding 'Key'
-> Key
-- ^ Table name for Map
-> TomlCodec map
internalTableMap emptyMap toListMap fromListMap unionsMap keyBiMap valCodec tableName =
Codec input output
where
-- tableNames :: TOML -> [Key]
-- tableNames t = applicableKey "group.*.dependencies" (tomlTables t)

getTable :: Key -> TomlEnv map
getTable lookupKey = \t -> case Prefix.lookup lookupKey $ tomlTables t of
Nothing -> Success emptyMap
Just toml' ->
let valKeys = HashMap.keys $ tomlPairs toml'
tableKeys = fmap (:|| []) $ HashMap.keys $ tomlTables toml'
tableArrayKey = HashMap.keys $ tomlTableArrays toml'
in fmap fromListMap $ for (valKeys <> tableKeys <> tableArrayKey) $ \key ->
whenLeftBiMapError key (forward keyBiMap key) $ \k ->
(k,) <$> codecRead (valCodec key) toml'

merge :: [Validation [TomlDecodeError] map] -> Validation [TomlDecodeError] map
merge validations =
case sequenceA validations of
Failure errors -> Failure errors
Success values -> Success $ unionsMap values

input :: TomlEnv map
input = \t -> do
let viableKeys = trace ("input.viableKeys:" ++ show (applicableKey "group.*.dependencies" (tomlTables t))) $ applicableKey "group.*.dependencies" (tomlTables t)
let maps = merge $ map (`getTable` t) viableKeys
maps

input' :: TomlEnv map
input' = \t -> case Prefix.lookup tableName $ tomlTables t of

Check failure on line 110 in src/Data/Toml/Extra.hs

View workflow job for this annotation

GitHub Actions / macOS-build

Defined but not used: ‘input'’
Nothing -> Success emptyMap
Just toml ->
let valKeys = HashMap.keys $ tomlPairs toml
tableKeys = fmap (:|| []) $ HashMap.keys $ tomlTables toml
tableArrayKey = HashMap.keys $ tomlTableArrays toml
in fmap fromListMap $ for (valKeys <> tableKeys <> tableArrayKey) $ \key ->
whenLeftBiMapError key (forward keyBiMap key) $ \k ->
(k,) <$> codecRead (valCodec key) toml

-- for tableNames $ \tableName' ->
-- case lookup tableName' tables of
-- Nothing -> pure Nothing
-- Just toml -> Just toml

-- maps :: [Maybe (Key, TOML)] <- for tableNames $ \tableName' ->
-- case lookup tableName' tables of
-- Nothing -> pure Nothing
-- Just toml -> pure . Just $ (tableName', toml)

-- Just toml -> trace ("input.toml: " ++ show toml) $ do
-- let valKeys = trace ("input.valKeys: " ++ show (HashMap.keys $ tomlPairs toml)) $ HashMap.keys $ tomlPairs toml
-- let tableKeys = trace ("input.tableKeys: " ++ show (HashMap.keys $ tomlTables toml)) $ fmap (:|| []) $ HashMap.keys $ tomlTables toml

-- for (valKeys <> tableKeys) $ \key -> do
-- whenLeftBiMapError key (forward keyBiMap key) $ \k -> trace ("input.whenLeftBiMapError.key: " ++ show key) $
-- (k,) <$> codecRead (valCodec key) toml

-- Success emptyMap

output :: map -> TomlState map
output m = do
mTable <- gets $ lookup tableName . tomlTables
let toml = fromMaybe mempty mTable
let (_, newToml) = unTomlState updateMapTable toml
m <$ modify (insertTable tableName newToml)
where
updateMapTable :: TomlState ()
updateMapTable = forM_ (toListMap m) $ \(k, v) -> case backward keyBiMap k of
Left _ -> empty
Right key -> codecWrite (valCodec key) v


lookupT :: Key -> Prefix.PrefixTree a -> Maybe a
lookupT lk (Prefix.Leaf k v) = trace ("input.lookupT.Prefix.Leaf => k:" ++ show k ++ " lk:" ++ show lk) $ if compareKey lk k then Just v else Nothing
lookupT lk (Prefix.Branch pref mv prefMap) = trace ("input.lookupT.Prefix.Branch => pref:" ++ show pref ++ " lk:" ++ show lk) $
case keysDiff' pref lk of
Equal -> trace ("input.lookupT.Prefix.Branch.Equal:") $ mv
NoPrefix -> trace ("input.lookupT.Prefix.Branch.NoPrefix:") $ Nothing
Diff _ _ _ -> trace ("input.lookupT.Prefix.Branch.Diff:") $ Nothing
SndIsPref _ -> trace ("input.lookupT.Prefix.Branch.SndIsPref:") $ Nothing
-- The first key is the prefix of the second one.
FstIsPref k -> trace ("input.lookupT.Prefix.Branch.FstIsPref: k:" ++ show k) $ lookup k prefMap

lookup :: Key -> Prefix.PrefixMap a -> Maybe a
lookup k@(p :|| _) prefMap = HashMap.lookup p prefMap >>= lookupT k



compareKey :: Key -> Key -> Bool
compareKey lhs rhs = lhs == rhs

keysDiff' :: Key -> Key -> KeysDiff
keysDiff' (x :|| xs) (y :|| ys)
| x == y = trace ("x <> y:" ++ show x ++ ".." ++ show y) $ listSame xs ys []
| x == "*" = trace ("x <> y:" ++ show x ++ ".." ++ show y) $ listSame xs ys []
| otherwise = trace ("x <> y:" ++ show x ++ ".." ++ show y) $ NoPrefix
where
listSame :: [Piece] -> [Piece] -> [Piece] -> KeysDiff
listSame [] [] _ = Equal
listSame [] (s:ss) _ = FstIsPref $ s :|| ss
listSame (f:fs) [] _ = SndIsPref $ f :|| fs
listSame (f:fs) (s:ss) pr =
if f == s
then listSame fs ss (pr ++ [f])
else Diff (x :|| pr) (f :|| fs) (s :|| ss)

compareKey' :: Key -> Key -> Bool
compareKey' (x :|| xs) (y :|| ys) = trace ("input.compareKey' => lhs:" ++ show x ++ " rhs:" ++ show y) $
if (x == "*" || x == y)
then do
case (mkKey xs, mkKey ys) of
(Just lhs', Just rhs') -> trace ("input.compareKey' => lhs':" ++ show lhs' ++ " rhs':" ++ show rhs') $ compareKey' lhs' rhs'
(Nothing, Nothing) -> True
_ -> False
else False

mkKey :: [Piece] -> Maybe Key
mkKey xs = Key <$> NE.nonEmpty xs



57 changes: 46 additions & 11 deletions src/Strategy/Python/Poetry.hs
Expand Up @@ -2,9 +2,14 @@ module Strategy.Python.Poetry (
discover,

-- * for testing only
graphFromLockFile,
findProjects,
analyze,
graphFromPyProjectAndLockFile,
setGraphDirectsFromPyproject,
PoetryProject (..),
PyProjectTomlFile (..),
PoetryLockFile (..),
ProjectDir (..),
) where

import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject)
Expand All @@ -18,7 +23,7 @@ import Data.Map (Map)
import Data.Map.Strict qualified as Map
import Data.Maybe (fromMaybe)
import Data.Text (Text)
import DepTypes (DepType (..), Dependency (..))
import DepTypes (DepType (..), Dependency (..), hydrateDepEnvs)
import Diag.Common (
MissingDeepDeps (MissingDeepDeps),
MissingEdges (MissingEdges),
Expand All @@ -30,7 +35,7 @@ import Discovery.Walk (
findFileNamed,
walkWithFilters',
)
import Effect.Logger (Logger, Pretty (pretty), logDebug)
import Effect.Logger (Logger, Pretty (pretty), logDebug, logStdout, logInfo)
import Effect.ReadFS (ReadFS, readContentsToml)
import GHC.Generics (Generic)
import Graphing (Graphing)
Expand All @@ -42,8 +47,10 @@ import Strategy.Python.Errors (
)
import Strategy.Python.Poetry.Common (getPoetryBuildBackend, logIgnoredDeps, pyProjectDeps, toCanonicalName, toMap)
import Strategy.Python.Poetry.PoetryLock (PackageName (..), PoetryLock (..), PoetryLockPackage (..), poetryLockCodec)
import Strategy.Python.Poetry.PyProject (PyProject (..), pyProjectCodec)
import Strategy.Python.Poetry.PyProject (PyProject (..), pyProjectCodec, allPoetryProductionDeps, allPoetryNonProductionDeps)
import Types (DependencyResults (..), DiscoveredProject (..), DiscoveredProjectType (PoetryProjectType), GraphBreadth (..))
import Text.Pretty.Simple (pShow)
import Data.Text.Lazy (toStrict)

newtype PyProjectTomlFile = PyProjectTomlFile {pyProjectTomlPath :: Path Abs File} deriving (Eq, Ord, Show, Generic)
newtype PoetryLockFile = PoetryLockFile {poetryLockPath :: Path Abs File} deriving (Eq, Ord, Show, Generic)
Expand Down Expand Up @@ -154,7 +161,8 @@ analyze PoetryProject{pyProjectToml, poetryLock} = do
Just lockPath -> do
poetryLockProject <- readContentsToml poetryLockCodec (poetryLockPath lockPath)
_ <- logIgnoredDeps pyproject (Just poetryLockProject)
graph <- context "Building dependency graph from pyproject.toml and poetry.lock" $ pure $ setGraphDirectsFromPyproject (graphFromLockFile poetryLockProject) pyproject
logInfo $ pretty $ pShow pyproject
graph <- context "Building dependency graph from pyproject.toml and poetry.lock" $ setGraphDirectsFromPyproject (graphFromPyProjectAndLockFile pyproject poetryLockProject) pyproject
pure $
DependencyResults
{ dependencyGraph = graph
Expand All @@ -179,22 +187,43 @@ analyze PoetryProject{pyProjectToml, poetryLock} = do
}

-- | Use a `pyproject.toml` to set the direct dependencies of a graph created from `poetry.lock`.
setGraphDirectsFromPyproject :: Graphing Dependency -> PyProject -> Graphing Dependency
setGraphDirectsFromPyproject graph pyproject = Graphing.promoteToDirect isDirect graph
setGraphDirectsFromPyproject :: (Has Logger sig m) => Graphing Dependency -> PyProject -> m (Graphing Dependency)
setGraphDirectsFromPyproject graph pyproject = do
-- let graph = hydrateDepEnvs graph'
logInfo $ pretty $ pShow directDeps
logInfo $ pretty $ pShow prodPkgNames
logInfo $ pretty $ pShow devPkgNames
logInfo $ pretty $ pShow graph'
pure graph'
where
graph' :: Graphing Dependency
graph' = hydrateDepEnvs $ Graphing.promoteToDirect isDirect graph

prodPkgNames :: [Text]
prodPkgNames = Map.keys $ allPoetryProductionDeps pyproject

devPkgNames :: [Text]
devPkgNames = Map.keys $ allPoetryNonProductionDeps pyproject

directDeps :: [Dependency]
directDeps = pyProjectDeps pyproject

-- Dependencies in `poetry.lock` are direct if they're specified in `pyproject.toml`.
-- `pyproject.toml` may use non canonical naming, when naming dependencies.
isDirect :: Dependency -> Bool
isDirect dep = case pyprojectPoetry pyproject of
Nothing -> False
Just _ -> any (\n -> toCanonicalName (dependencyName n) == toCanonicalName (dependencyName dep)) $ pyProjectDeps pyproject
Just _ -> any (\n -> toCanonicalName (dependencyName n) == toCanonicalName (dependencyName dep)) directDeps

-- | Using a Poetry lockfile, build the graph of packages.
-- The resulting graph contains edges, but does not distinguish between direct and deep dependencies,
-- since `poetry.lock` does not indicate which dependencies are direct.
graphFromLockFile :: PoetryLock -> Graphing Dependency
graphFromLockFile poetryLock = Graphing.gmap pkgNameToDependency (edges <> Graphing.deeps pkgsNoDeps)
graphFromPyProjectAndLockFile :: PyProject -> PoetryLock -> Graphing Dependency
graphFromPyProjectAndLockFile pyProject poetryLock = graph
where
graph :: Graphing Dependency
graph = Graphing.gmap pkgNameToDependency (edges <> Graphing.deeps pkgsNoDeps)

pkgs :: [PoetryLockPackage]
pkgs = poetryLockPackages poetryLock

Expand All @@ -217,7 +246,13 @@ graphFromLockFile poetryLock = Graphing.gmap pkgNameToDependency (edges <> Graph
canonicalPkgName name = PackageName . toCanonicalName $ unPackageName name

mapOfDependency :: Map PackageName Dependency
mapOfDependency = toMap pkgs
mapOfDependency = toMap prodPkgNames devPkgNames pkgs

prodPkgNames :: [PackageName]
prodPkgNames = PackageName <$> Map.keys (allPoetryProductionDeps pyProject)

devPkgNames :: [PackageName]
devPkgNames = PackageName <$> Map.keys (allPoetryNonProductionDeps pyProject)

-- Pip packages are [case insensitive](https://www.python.org/dev/peps/pep-0508/#id21), but poetry.lock may use
-- non-canonical name for reference. Try to lookup with provided name, otherwise fallback to canonical naming.
Expand Down

0 comments on commit 7287894

Please sign in to comment.