Integration tests (#138)

Makefile

@@ -21,7 +21,7 @@ ctags:
 		> tags
 
 build-example: $(OUTPUT)
-	npx spago bundle-app --path $(example) --to $(OUTPUT)/example.js --minify
+	npx spago bundle-app --path $(example) --to $(OUTPUT)/example.js --minify --platform=node
 
 run-example: build-example
 	npx spago run --path $(example)

bower.json

@@ -33,11 +33,20 @@
         "purescript-lists": "^v7.0.0",
         "purescript-maybe": "^v6.0.0",
         "purescript-newtype": "^v5.0.0",
+        "purescript-node-buffer": "^v8.0.0",
+        "purescript-node-child-process": "^v9.0.0",
+        "purescript-node-fs": "^v8.2.0",
+        "purescript-node-fs-aff": "^9.2.0",
+        "purescript-node-os": "^v4.0.0",
+        "purescript-node-process": "^v10.0.0",
+        "purescript-node-streams": "^v7.0.0",
         "purescript-now": "^v6.0.0",
+        "purescript-numbers": "^v9.0.1",
         "purescript-ordered-collections": "^v3.0.0",
         "purescript-parallel": "^v6.0.0",
         "purescript-pipes": "^v8.0.0",
         "purescript-prelude": "^v6.0.1",
+        "purescript-refs": "^v6.0.0",
         "purescript-strings": "^v6.0.1",
         "purescript-tailrec": "^v6.1.0",
         "purescript-transformers": "^v6.0.0",

integration-tests/README.md

@@ -0,0 +1,12 @@
+# Integration tests
+
+This directory contains fixtures/assets for integration tests, in which we run
+`spago test` on an example program and check that its output is as expected.
+
+`env-template` contains a skeleton of a PureScript project that will be used as
+context for executing examples.
+
+`cases` contains multiple subdirectories, each representing a test case. Every
+test case consists of a `Main.purs` file (containing the test program to be run
+via `spago test`) and an `output.txt` file (containing the program's expected
+output).

integration-tests/cases/01-all-passing/Main.purs

@@ -0,0 +1,27 @@
+module Test.Main where
+
+import Prelude
+
+import Effect (Effect)
+import Effect.Aff (Milliseconds(..), delay, launchAff_)
+import Test.Spec (describe, it)
+import Test.Spec.Assertions (shouldEqual)
+import Test.Spec.Reporter (specReporter)
+import Test.Spec.Runner (runSpec)
+
+main :: Effect Unit
+main = launchAff_ $ runSpec [specReporter] do
+  it "one" $
+    5 `shouldEqual` (3 + 2)
+
+  describe "two" do
+    it "first" $ pure unit
+    it "second" $ pure unit
+    it "third" $ delay (Milliseconds 20.0)
+
+  describe "three" do
+    describe "1" do
+      it "uno" $ pure unit
+      it "dos" $ pure unit
+    describe "2" do
+      it "ein" $ pure unit

integration-tests/cases/01-all-passing/output.txt

@@ -0,0 +1,12 @@
+ [32m✓︎ [0m[2mone[0m
+ two
+  [32m✓︎ [0m[2mfirst[0m
+  [32m✓︎ [0m[2msecond[0m
+  [32m✓︎ [0m[2mthird[0m
+ three
+  1
+   [32m✓︎ [0m[2muno[0m
+   [32m✓︎ [0m[2mdos[0m
+  2
+   [32m✓︎ [0m[2mein[0m
+[32m7 passing[0m

integration-tests/cases/02-all-failing/Main.purs

@@ -0,0 +1,26 @@
+module Test.Main where
+
+import Prelude
+
+import Effect (Effect)
+import Effect.Aff (launchAff_)
+import Test.Spec (describe, it)
+import Test.Spec.Assertions (fail, shouldEqual)
+import Test.Spec.Reporter (specReporter)
+import Test.Spec.Runner (runSpec)
+
+main :: Effect Unit
+main = launchAff_ $ runSpec [specReporter] do
+  it "one" $
+    5 `shouldEqual` (3 + 3)
+
+  describe "two" do
+    it "first" $ fail "boom"
+    it "second" $ fail "crash"
+
+  describe "three" do
+    describe "1" do
+      it "uno" $ fail "ohmigawd"
+      it "dos" $ fail "aaargh!"
+    describe "2" do
+      it "ein" $ fail "die"

integration-tests/cases/02-all-failing/output.txt

@@ -0,0 +1,24 @@
+ [31m1) one[0m
+ two
+  [31m2) first[0m
+  [31m3) second[0m
+ three
+  1
+   [31m4) uno[0m
+   [31m5) dos[0m
+  2
+   [31m6) ein[0m
+[31m6 failed[0m
+
+1) one
+[31m  5 ≠ 6[0m
+2) two first
+[31m  boom[0m
+3) two second
+[31m  crash[0m
+4) three 1 uno
+[31m  ohmigawd[0m
+5) three 1 dos
+[31m  aaargh![0m
+6) three 2 ein
+[31m  die[0m

integration-tests/cases/03-some-passing/Main.purs

@@ -0,0 +1,26 @@
+module Test.Main where
+
+import Prelude
+
+import Effect (Effect)
+import Effect.Aff (launchAff_)
+import Test.Spec (describe, it)
+import Test.Spec.Assertions (fail, shouldEqual)
+import Test.Spec.Reporter (specReporter)
+import Test.Spec.Runner (runSpec)
+
+main :: Effect Unit
+main = launchAff_ $ runSpec [specReporter] do
+  it "one" $
+    5 `shouldEqual` (3 + 3)
+
+  describe "two" do
+    it "first" $ pure unit
+    it "second" $ fail "crash"
+
+  describe "three" do
+    describe "1" do
+      it "uno" $ fail "boom"
+      it "dos" $ pure unit
+    describe "2" do
+      it "ein" $ pure unit

integration-tests/cases/03-some-passing/output.txt

@@ -0,0 +1,19 @@
+ [31m1) one[0m
+ two
+  [32m✓︎ [0m[2mfirst[0m
+  [31m2) second[0m
+ three
+  1
+   [31m3) uno[0m
+   [32m✓︎ [0m[2mdos[0m
+  2
+   [32m✓︎ [0m[2mein[0m
+[32m3 passing[0m
+[31m3 failed[0m
+
+1) one
+[31m  5 ≠ 6[0m
+2) two second
+[31m  crash[0m
+3) three 1 uno
+[31m  boom[0m

integration-tests/cases/04-fail-fast-with-timeout/Main.purs

@@ -0,0 +1,26 @@
+module Test.Main where
+
+import Prelude
+
+import Data.Maybe (Maybe(..))
+import Effect (Effect)
+import Effect.Aff (Milliseconds(..), delay, launchAff_)
+import Test.Spec (it)
+import Test.Spec.Assertions (shouldEqual)
+import Test.Spec.Config (defaultConfig)
+import Test.Spec.Reporter (specReporter)
+import Test.Spec.Runner (runSpec')
+
+main :: Effect Unit
+main = launchAff_ $ runSpec' config [specReporter] do
+  it "passes quickly" $
+    5 `shouldEqual` (3 + 2)
+
+  it "times out" $
+    delay (Milliseconds 15.0)
+
+  it "shouldn't get to run" $
+    2 `shouldEqual` 3
+
+  where
+    config = defaultConfig { failFast = true, timeout = Just $ Milliseconds 10.0 }

integration-tests/cases/04-fail-fast-with-timeout/output.txt

@@ -0,0 +1,6 @@
+ [32m✓︎ [0m[2mpasses quickly[0m
+ [31m1) times out[0m
+[31m1 failed[0m
+
+1) times out
+[31m  test timed out after 10ms[0m

integration-tests/cases/05-fail-fast-due-to-test-failure/Main.purs

@@ -0,0 +1,25 @@
+module Test.Main where
+
+import Prelude
+
+import Effect (Effect)
+import Effect.Aff (launchAff_)
+import Test.Spec (it)
+import Test.Spec.Assertions (fail, shouldEqual)
+import Test.Spec.Config (defaultConfig)
+import Test.Spec.Reporter (specReporter)
+import Test.Spec.Runner (runSpec')
+
+main :: Effect Unit
+main = launchAff_ $ runSpec' config [specReporter] do
+  it "passes" $
+    5 `shouldEqual` (3 + 2)
+
+  it "fails" $
+    fail "This is a failing test"
+
+  it "shouldn't get to run" $
+    2 `shouldEqual` 3
+
+  where
+    config = defaultConfig { failFast = true }

integration-tests/cases/05-fail-fast-due-to-test-failure/output.txt

@@ -0,0 +1,6 @@
+ [32m✓︎ [0m[2mpasses[0m
+ [31m1) fails[0m
+[31m1 failed[0m
+
+1) fails
+[31m  This is a failing test[0m

integration-tests/env-template/packages.dhall

@@ -0,0 +1,7 @@
+let packages =
+      https://github.com/purescript/package-sets/releases/download/psc-0.15.9-20230629/packages.dhall
+
+    with
+      spec = (SPEC_REPO_PATH/spago.dhall as Location)
+
+in packages

integration-tests/env-template/spago.dhall

@@ -0,0 +1,5 @@
+{ name = ""
+, dependencies = ["aff", "effect", "maybe", "prelude", "spec"]
+, packages = ./packages.dhall
+, sources = [ "test/**/*.purs" ]
+}

packages.dhall

@@ -1,5 +1,5 @@
 let upstream =
-      https://github.com/purescript/package-sets/releases/download/psc-0.15.4-20221229/packages.dhall
-        sha256:a6af1091425f806ec0da34934bb6c0ab0ac1598620bbcbb60a7d463354e7d87c
+      https://github.com/purescript/package-sets/releases/download/psc-0.15.9-20230629/packages.dhall
+        sha256:f91d36c7e4793fe4d7e042c57fef362ff3f9e9ba88454cd38686701e30bf545a
 
 in  upstream

src/Test/Spec/Config.purs

@@ -0,0 +1,34 @@
+module Test.Spec.Config
+  ( Config
+  , defaultConfig
+  )
+  where
+
+import Prelude
+
+import Data.Maybe (Maybe(..))
+import Data.Time.Duration (Milliseconds(..))
+
+type Config =
+  { slow :: Milliseconds
+  -- ^ Threshold of time beyond which a test is considered "slow".
+
+  , timeout :: Maybe Milliseconds
+  -- ^ An optional timeout, applied to each individual test. When omitted, tests
+  -- are allowed to run forever.
+
+  , exit :: Boolean
+  -- ^ When `true`, the runner will exit the Node process after running tests.
+  -- If `false`, the runner will merely return test results.
+
+  , failFast :: Boolean
+  -- ^ When `true`, first failed test stops the whole run.
+  }
+
+defaultConfig :: Config
+defaultConfig =
+  { slow: Milliseconds 75.0
+  , timeout: Just $ Milliseconds 2000.0
+  , exit: true
+  , failFast: false
+  }

src/Test/Spec/Reporter/Base.purs

@@ -85,7 +85,7 @@ scanWithStateM step begin = do
   go x
   where
     go x = do
-        a  <- await
+        a <- await
         yield a
         x' <- lift (step x a)
         go $ x'

src/Test/Spec/Runner.purs

@@ -3,10 +3,9 @@ module Test.Spec.Runner
   , runSpecT
   , runSpec
   , runSpec'
-  , defaultConfig
-  , Config
   , TestEvents
   , Reporter
+  , module Test.Spec.Config
   ) where
 
 import Prelude
@@ -37,6 +36,7 @@ import Pipes.Core (Producer, Pipe, (//>))
 import Pipes.Core (runEffect, runEffectRec) as P
 import Prim.TypeError (class Warn, Text)
 import Test.Spec (Item(..), Spec, SpecT, SpecTree, Tree(..), collect)
+import Test.Spec.Config (Config, defaultConfig)
 import Test.Spec.Console as Console
 import Test.Spec.Result (Result(..))
 import Test.Spec.Runner.Event (Event, Execution(..))
@@ -49,30 +49,6 @@ import Test.Spec.Tree (Path, PathItem(..), countTests, isAllParallelizable, pare
 
 foreign import exit :: Int -> Effect Unit
 
-type Config =
-  { slow :: Milliseconds
-  -- ^ Threshold of time beyond which a test is considered "slow".
-
-  , timeout :: Maybe Milliseconds
-  -- ^ An optional timeout, applied to each individual test. When omitted, tests
-  -- are allowed to run forever.
-
-  , exit :: Boolean
-  -- ^ When `true`, the runner will exit the Node process after running tests.
-  -- If `false`, the runner will merely return test results.
-
-  , failFast :: Boolean
-  -- ^ When `true`, first failed test stops the whole run.
-  }
-
-defaultConfig :: Config
-defaultConfig =
-  { slow: Milliseconds 75.0
-  , timeout: Just $ Milliseconds 2000.0
-  , exit: true
-  , failFast: false
-  }
-
 makeTimeout
   :: Milliseconds
   -> Aff Unit
@@ -231,7 +207,7 @@ runSpec
   :: Array Reporter
   -> Spec Unit
   -> Aff Unit
-runSpec = runSpec' defaultConfig
+runSpec reporters spec = runSpec' defaultConfig reporters spec
 
 runSpec'
   :: Config

test.dhall

@@ -2,5 +2,15 @@ let config = ./spago.dhall
 
 in config // {
   sources = config.sources # [ "test/**/*.purs" ],
-  dependencies = config.dependencies # [ "console" ]
+  dependencies = config.dependencies #
+    [ "console"
+    , "node-buffer"
+    , "node-child-process"
+    , "node-fs"
+    , "node-fs-aff"
+    , "node-os"
+    , "node-process"
+    , "node-streams"
+    , "refs"
+    ]
 }