-
Notifications
You must be signed in to change notification settings - Fork 114
/
NixLanguageTests.hs
235 lines (206 loc) · 8.42 KB
/
NixLanguageTests.hs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
module NixLanguageTests (genTests) where
import Nix.Prelude
import Control.Exception
import GHC.Err ( errorWithoutStackTrace )
import Control.Monad.ST
import Data.List ( delete )
import Data.List.Split ( splitOn )
import qualified Data.Map as Map
import qualified Data.Set as Set
import qualified Data.String as String
import qualified Data.Text as Text
import Data.Time
import GHC.Exts
import Nix.Lint
import Nix.Options
import Nix.Options.Parser
import Nix.Parser
import Nix.Pretty
import Nix.String
import Nix.XML
import qualified Options.Applicative as Opts
import System.Environment ( setEnv )
import System.FilePath.Glob ( compile
, globDir1
)
import Test.Tasty
import Test.Tasty.HUnit
import TestCommon
{-
From (git://nix)/tests/lang.sh we see that
lang/parse-fail-*.nix -> parsing should fail
lang/parse-okay-*.nix -> parsing should succeed
lang/eval-fail-*.nix -> eval should fail
lang/eval-okay-*.{nix,xml} -> eval should succeed,
xml dump should be the same as the .xml
lang/eval-okay-*.{nix,exp} -> eval should succeed,
plain text output should be the same as the .exp
lang/eval-okay-*.{nix,exp,flags} -> eval should succeed,
plain text output should be the same as the .exp,
pass the extra flags to nix-instantiate
NIX_PATH=lang/dir3:lang/dir4 should be in the environment of all
eval-okay-*.nix evaluations
TEST_VAR=foo should be in all the environments # for eval-okay-getenv.nix
-}
groupBy :: Ord k => (v -> k) -> [v] -> Map k [v]
groupBy key = Map.fromListWith (<>) . fmap (key &&& pure)
-- | New tests, which have never yet passed. Once any of these is passing,
-- please remove it from this list. Do not add tests to this list if they have
-- previously passed.
newFailingTests :: Set String
newFailingTests = Set.fromList
[ "eval-okay-fromTOML"
, "eval-okay-zipAttrsWith"
, "eval-okay-tojson"
, "eval-okay-search-path"
, "eval-okay-sort"
, "eval-okay-path-antiquotation"
, "eval-okay-getattrpos-functionargs"
, "eval-okay-attrs6"
]
-- | Upstream tests that test cases that HNix disaded as a misfeature that is used so rarely
-- that it more effective to fix it & lint it out of existance.
deprecatedRareNixQuirkTests :: Set String
deprecatedRareNixQuirkTests = Set.fromList
[ -- A rare quirk of Nix that is proper to fix&enforce then to support (see git commit history)
"eval-okay-strings-as-attrs-names"
-- Nix upstream removed this test altogether
, "eval-okay-hash"
]
genTests :: IO TestTree
genTests =
do
testFiles <- getTestFiles
let
testsGroupedByName :: Map Path [Path]
testsGroupedByName = groupBy (takeFileName . dropExtensions) testFiles
testsGroupedByTypeThenName :: Map [String] [(Path, [Path])]
testsGroupedByTypeThenName = groupBy testType $ Map.toList testsGroupedByName
testTree :: [TestTree]
testTree = mkTestGroup <$> Map.toList testsGroupedByTypeThenName
pure $
localOption
(mkTimeout 2000000)
$ testGroup
"Nix (upstream) language tests"
testTree
where
getTestFiles :: IO [Path]
getTestFiles = sortTestFiles <$> collectTestFiles
where
collectTestFiles :: IO [Path]
collectTestFiles = coerce (globDir1 (compile "*-*-*.*") nixTestDir)
sortTestFiles :: [Path] -> [Path]
sortTestFiles =
sort
-- Disabling the not yet done tests cases.
. filter withoutDisabledTests
. filter withoutXml
where
withoutDisabledTests :: Path -> Bool
withoutDisabledTests = (`Set.notMember` (newFailingTests `Set.union` deprecatedRareNixQuirkTests)) . takeBaseName
withoutXml :: Path -> Bool
withoutXml = (/= ".xml") . takeExtension
testType :: (Path, b) -> [String]
testType (fullpath, _files) = coerce (take 2 . splitOn "-") $ takeFileName fullpath
mkTestGroup :: ([String], [(Path, [Path])]) -> TestTree
mkTestGroup (tType, tests) =
testGroup (String.unwords tType) $ mkTestCase <$> tests
where
mkTestCase :: (Path, [Path]) -> TestTree
mkTestCase (basename, files) =
testCase
(coerce $ takeFileName basename)
$ do
time <- liftIO getCurrentTime
let opts = defaultOptions time
case tType of
["parse", "okay"] -> assertParse opts $ the files
["parse", "fail"] -> assertParseFail opts $ the files
["eval" , "okay"] -> assertEval opts files
["eval" , "fail"] -> assertEvalFail $ the files
_ -> fail $ "Unexpected: " <> show tType
assertParse :: Options -> Path -> Assertion
assertParse _opts file =
either
(\ err -> assertFailure $ "Failed to parse " <> coerce file <> ":\n" <> show err)
(const stub) -- pure $! runST $ void $ lint opts expr
=<< parseNixFileLoc file
assertParseFail :: Options -> Path -> Assertion
assertParseFail opts file =
(`catch` \(_ :: SomeException) -> stub) $
either
(const stub)
(\ expr ->
do
_ <- pure $! runST $ void $ lint opts expr
assertFailure $ "Unexpected success parsing `" <> coerce file <> ":\nParsed value: " <> show expr
)
=<< parseNixFileLoc file
assertLangOk :: Options -> Path -> Assertion
assertLangOk opts fileBaseName =
do
actual <- printNix <$> hnixEvalFile opts (addNixExt fileBaseName)
expected <- read fileBaseName ".exp"
assertEqual mempty expected (actual <> "\n")
assertLangOkXml :: Options -> Path -> Assertion
assertLangOkXml opts fileBaseName =
do
actual <- ignoreContext . toXML <$> hnixEvalFile opts (addNixExt fileBaseName)
expected <- read fileBaseName ".exp.xml"
assertEqual mempty expected actual
assertEval :: Options -> [Path] -> Assertion
assertEval _opts files =
do
time <- liftIO getCurrentTime
let opts = defaultOptions time
case delete ".nix" $ sort $ fromString @Text . takeExtensions <$> files of
[] -> void $ hnixEvalFile opts $ addNixExt name
[".exp" ] -> assertLangOk opts name
[".exp.xml" ] -> assertLangOkXml opts name
[".exp.disabled" ] -> stub
[".exp-disabled" ] -> stub
[".exp", ".flags"] ->
do
liftIO $ setEnv "NIX_PATH" "lang/dir4:lang/dir5"
flags <- read name ".flags"
let
flags' :: Text
flags' =
bool
id
Text.init
(Text.last flags == '\n')
flags
case runParserGetResult time flags' of
Opts.Failure err -> errorWithoutStackTrace $ "Error parsing flags from " <> coerce name <> ".flags: " <> show err
Opts.CompletionInvoked _ -> fail "unused"
Opts.Success opts' -> assertLangOk opts' name
_ -> assertFailure $ "Unknown test type " <> show files
where
runParserGetResult :: UTCTime -> Text -> Opts.ParserResult Options
runParserGetResult time flags' =
Opts.execParserPure
Opts.defaultPrefs
(nixOptionsInfo time)
(fmap toString $ fixup $ Text.splitOn " " flags')
name :: Path
name = coerce nixTestDir <> the (takeFileName . dropExtensions <$> files)
fixup :: [Text] -> [Text]
fixup ("--arg" : x : y : rest) = "--arg" : (x <> "=" <> y) : fixup rest
fixup ("--argstr" : x : y : rest) = "--argstr" : (x <> "=" <> y) : fixup rest
fixup (x : rest) = x : fixup rest
fixup [] = mempty
assertEvalFail :: Path -> Assertion
assertEvalFail file =
(`catch` (\(_ :: SomeException) -> stub)) $
do
time <- liftIO getCurrentTime
evalResult <- printNix <$> hnixEvalFile (defaultOptions time) file
evalResult `seq` assertFailure $ "File: ''" <> coerce file <> "'' should not evaluate.\nThe evaluation result was `" <> toString evalResult <> "`."
nixTestDir :: FilePath
nixTestDir = "data/nix/tests/lang/"
addNixExt :: Path -> Path
addNixExt path = addExtension path ".nix"
read :: Path -> String -> IO Text
read path ext = readFile $ addExtension path ext