diff --git a/Makefile b/Makefile index 1d1d788..460ab40 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ ctags: > tags build-example: $(OUTPUT) - npx spago bundle-app --path $(example) --to $(OUTPUT)/example.js --minify + npx spago bundle-app --path $(example) --to $(OUTPUT)/example.js --minify --platform=node run-example: build-example npx spago run --path $(example) diff --git a/bower.json b/bower.json index 9fbb0dd..69de424 100644 --- a/bower.json +++ b/bower.json @@ -33,11 +33,20 @@ "purescript-lists": "^v7.0.0", "purescript-maybe": "^v6.0.0", "purescript-newtype": "^v5.0.0", + "purescript-node-buffer": "^v8.0.0", + "purescript-node-child-process": "^v9.0.0", + "purescript-node-fs": "^v8.2.0", + "purescript-node-fs-aff": "^9.2.0", + "purescript-node-os": "^v4.0.0", + "purescript-node-process": "^v10.0.0", + "purescript-node-streams": "^v7.0.0", "purescript-now": "^v6.0.0", + "purescript-numbers": "^v9.0.1", "purescript-ordered-collections": "^v3.0.0", "purescript-parallel": "^v6.0.0", "purescript-pipes": "^v8.0.0", "purescript-prelude": "^v6.0.1", + "purescript-refs": "^v6.0.0", "purescript-strings": "^v6.0.1", "purescript-tailrec": "^v6.1.0", "purescript-transformers": "^v6.0.0", diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 0000000..a202bc3 --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,12 @@ +# Integration tests + +This directory contains fixtures/assets for integration tests, in which we run +`spago test` on an example program and check that its output is as expected. + +`env-template` contains a skeleton of a PureScript project that will be used as +context for executing examples. + +`cases` contains multiple subdirectories, each representing a test case. Every +test case consists of a `Main.purs` file (containing the test program to be run +via `spago test`) and an `output.txt` file (containing the program's expected +output). diff --git a/integration-tests/cases/01-all-passing/Main.purs b/integration-tests/cases/01-all-passing/Main.purs new file mode 100644 index 0000000..9852b50 --- /dev/null +++ b/integration-tests/cases/01-all-passing/Main.purs @@ -0,0 +1,27 @@ +module Test.Main where + +import Prelude + +import Effect (Effect) +import Effect.Aff (Milliseconds(..), delay, launchAff_) +import Test.Spec (describe, it) +import Test.Spec.Assertions (shouldEqual) +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (runSpec) + +main :: Effect Unit +main = launchAff_ $ runSpec [specReporter] do + it "one" $ + 5 `shouldEqual` (3 + 2) + + describe "two" do + it "first" $ pure unit + it "second" $ pure unit + it "third" $ delay (Milliseconds 20.0) + + describe "three" do + describe "1" do + it "uno" $ pure unit + it "dos" $ pure unit + describe "2" do + it "ein" $ pure unit diff --git a/integration-tests/cases/01-all-passing/output.txt b/integration-tests/cases/01-all-passing/output.txt new file mode 100644 index 0000000..d552546 --- /dev/null +++ b/integration-tests/cases/01-all-passing/output.txt @@ -0,0 +1,12 @@ + [32m✓︎ [0m[2mone[0m + two + [32m✓︎ [0m[2mfirst[0m + [32m✓︎ [0m[2msecond[0m + [32m✓︎ [0m[2mthird[0m + three + 1 + [32m✓︎ [0m[2muno[0m + [32m✓︎ [0m[2mdos[0m + 2 + [32m✓︎ [0m[2mein[0m +[32m7 passing[0m diff --git a/integration-tests/cases/02-all-failing/Main.purs b/integration-tests/cases/02-all-failing/Main.purs new file mode 100644 index 0000000..c9325dc --- /dev/null +++ b/integration-tests/cases/02-all-failing/Main.purs @@ -0,0 +1,26 @@ +module Test.Main where + +import Prelude + +import Effect (Effect) +import Effect.Aff (launchAff_) +import Test.Spec (describe, it) +import Test.Spec.Assertions (fail, shouldEqual) +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (runSpec) + +main :: Effect Unit +main = launchAff_ $ runSpec [specReporter] do + it "one" $ + 5 `shouldEqual` (3 + 3) + + describe "two" do + it "first" $ fail "boom" + it "second" $ fail "crash" + + describe "three" do + describe "1" do + it "uno" $ fail "ohmigawd" + it "dos" $ fail "aaargh!" + describe "2" do + it "ein" $ fail "die" diff --git a/integration-tests/cases/02-all-failing/output.txt b/integration-tests/cases/02-all-failing/output.txt new file mode 100644 index 0000000..a3a7dd4 --- /dev/null +++ b/integration-tests/cases/02-all-failing/output.txt @@ -0,0 +1,24 @@ + [31m1) one[0m + two + [31m2) first[0m + [31m3) second[0m + three + 1 + [31m4) uno[0m + [31m5) dos[0m + 2 + [31m6) ein[0m +[31m6 failed[0m + +1) one +[31m 5 ≠ 6[0m +2) two first +[31m boom[0m +3) two second +[31m crash[0m +4) three 1 uno +[31m ohmigawd[0m +5) three 1 dos +[31m aaargh![0m +6) three 2 ein +[31m die[0m diff --git a/integration-tests/cases/03-some-passing/Main.purs b/integration-tests/cases/03-some-passing/Main.purs new file mode 100644 index 0000000..2dd99e3 --- /dev/null +++ b/integration-tests/cases/03-some-passing/Main.purs @@ -0,0 +1,26 @@ +module Test.Main where + +import Prelude + +import Effect (Effect) +import Effect.Aff (launchAff_) +import Test.Spec (describe, it) +import Test.Spec.Assertions (fail, shouldEqual) +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (runSpec) + +main :: Effect Unit +main = launchAff_ $ runSpec [specReporter] do + it "one" $ + 5 `shouldEqual` (3 + 3) + + describe "two" do + it "first" $ pure unit + it "second" $ fail "crash" + + describe "three" do + describe "1" do + it "uno" $ fail "boom" + it "dos" $ pure unit + describe "2" do + it "ein" $ pure unit diff --git a/integration-tests/cases/03-some-passing/output.txt b/integration-tests/cases/03-some-passing/output.txt new file mode 100644 index 0000000..3e8d48a --- /dev/null +++ b/integration-tests/cases/03-some-passing/output.txt @@ -0,0 +1,19 @@ + [31m1) one[0m + two + [32m✓︎ [0m[2mfirst[0m + [31m2) second[0m + three + 1 + [31m3) uno[0m + [32m✓︎ [0m[2mdos[0m + 2 + [32m✓︎ [0m[2mein[0m +[32m3 passing[0m +[31m3 failed[0m + +1) one +[31m 5 ≠ 6[0m +2) two second +[31m crash[0m +3) three 1 uno +[31m boom[0m diff --git a/integration-tests/cases/04-fail-fast-with-timeout/Main.purs b/integration-tests/cases/04-fail-fast-with-timeout/Main.purs new file mode 100644 index 0000000..1e5248f --- /dev/null +++ b/integration-tests/cases/04-fail-fast-with-timeout/Main.purs @@ -0,0 +1,26 @@ +module Test.Main where + +import Prelude + +import Data.Maybe (Maybe(..)) +import Effect (Effect) +import Effect.Aff (Milliseconds(..), delay, launchAff_) +import Test.Spec (it) +import Test.Spec.Assertions (shouldEqual) +import Test.Spec.Config (defaultConfig) +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (runSpec') + +main :: Effect Unit +main = launchAff_ $ runSpec' config [specReporter] do + it "passes quickly" $ + 5 `shouldEqual` (3 + 2) + + it "times out" $ + delay (Milliseconds 15.0) + + it "shouldn't get to run" $ + 2 `shouldEqual` 3 + + where + config = defaultConfig { failFast = true, timeout = Just $ Milliseconds 10.0 } diff --git a/integration-tests/cases/04-fail-fast-with-timeout/output.txt b/integration-tests/cases/04-fail-fast-with-timeout/output.txt new file mode 100644 index 0000000..e345d59 --- /dev/null +++ b/integration-tests/cases/04-fail-fast-with-timeout/output.txt @@ -0,0 +1,6 @@ + [32m✓︎ [0m[2mpasses quickly[0m + [31m1) times out[0m +[31m1 failed[0m + +1) times out +[31m test timed out after 10ms[0m diff --git a/integration-tests/cases/05-fail-fast-due-to-test-failure/Main.purs b/integration-tests/cases/05-fail-fast-due-to-test-failure/Main.purs new file mode 100644 index 0000000..15649d4 --- /dev/null +++ b/integration-tests/cases/05-fail-fast-due-to-test-failure/Main.purs @@ -0,0 +1,25 @@ +module Test.Main where + +import Prelude + +import Effect (Effect) +import Effect.Aff (launchAff_) +import Test.Spec (it) +import Test.Spec.Assertions (fail, shouldEqual) +import Test.Spec.Config (defaultConfig) +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (runSpec') + +main :: Effect Unit +main = launchAff_ $ runSpec' config [specReporter] do + it "passes" $ + 5 `shouldEqual` (3 + 2) + + it "fails" $ + fail "This is a failing test" + + it "shouldn't get to run" $ + 2 `shouldEqual` 3 + + where + config = defaultConfig { failFast = true } diff --git a/integration-tests/cases/05-fail-fast-due-to-test-failure/output.txt b/integration-tests/cases/05-fail-fast-due-to-test-failure/output.txt new file mode 100644 index 0000000..375f28f --- /dev/null +++ b/integration-tests/cases/05-fail-fast-due-to-test-failure/output.txt @@ -0,0 +1,6 @@ + [32m✓︎ [0m[2mpasses[0m + [31m1) fails[0m +[31m1 failed[0m + +1) fails +[31m This is a failing test[0m diff --git a/integration-tests/env-template/packages.dhall b/integration-tests/env-template/packages.dhall new file mode 100644 index 0000000..46c4f0f --- /dev/null +++ b/integration-tests/env-template/packages.dhall @@ -0,0 +1,7 @@ +let packages = + https://github.com/purescript/package-sets/releases/download/psc-0.15.9-20230629/packages.dhall + + with + spec = (SPEC_REPO_PATH/spago.dhall as Location) + +in packages diff --git a/integration-tests/env-template/spago.dhall b/integration-tests/env-template/spago.dhall new file mode 100644 index 0000000..d6f0163 --- /dev/null +++ b/integration-tests/env-template/spago.dhall @@ -0,0 +1,5 @@ +{ name = "" +, dependencies = ["aff", "effect", "maybe", "prelude", "spec"] +, packages = ./packages.dhall +, sources = [ "test/**/*.purs" ] +} diff --git a/packages.dhall b/packages.dhall index 2aca328..1e51cff 100644 --- a/packages.dhall +++ b/packages.dhall @@ -1,5 +1,5 @@ let upstream = - https://github.com/purescript/package-sets/releases/download/psc-0.15.4-20221229/packages.dhall - sha256:a6af1091425f806ec0da34934bb6c0ab0ac1598620bbcbb60a7d463354e7d87c + https://github.com/purescript/package-sets/releases/download/psc-0.15.9-20230629/packages.dhall + sha256:f91d36c7e4793fe4d7e042c57fef362ff3f9e9ba88454cd38686701e30bf545a in upstream diff --git a/src/Test/Spec/Config.purs b/src/Test/Spec/Config.purs new file mode 100644 index 0000000..4ae8a43 --- /dev/null +++ b/src/Test/Spec/Config.purs @@ -0,0 +1,34 @@ +module Test.Spec.Config + ( Config + , defaultConfig + ) + where + +import Prelude + +import Data.Maybe (Maybe(..)) +import Data.Time.Duration (Milliseconds(..)) + +type Config = + { slow :: Milliseconds + -- ^ Threshold of time beyond which a test is considered "slow". + + , timeout :: Maybe Milliseconds + -- ^ An optional timeout, applied to each individual test. When omitted, tests + -- are allowed to run forever. + + , exit :: Boolean + -- ^ When `true`, the runner will exit the Node process after running tests. + -- If `false`, the runner will merely return test results. + + , failFast :: Boolean + -- ^ When `true`, first failed test stops the whole run. + } + +defaultConfig :: Config +defaultConfig = + { slow: Milliseconds 75.0 + , timeout: Just $ Milliseconds 2000.0 + , exit: true + , failFast: false + } diff --git a/src/Test/Spec/Reporter/Base.purs b/src/Test/Spec/Reporter/Base.purs index 5547334..910f039 100644 --- a/src/Test/Spec/Reporter/Base.purs +++ b/src/Test/Spec/Reporter/Base.purs @@ -85,7 +85,7 @@ scanWithStateM step begin = do go x where go x = do - a <- await + a <- await yield a x' <- lift (step x a) go $ x' diff --git a/src/Test/Spec/Runner.purs b/src/Test/Spec/Runner.purs index 2f92269..baaf9c1 100644 --- a/src/Test/Spec/Runner.purs +++ b/src/Test/Spec/Runner.purs @@ -3,10 +3,9 @@ module Test.Spec.Runner , runSpecT , runSpec , runSpec' - , defaultConfig - , Config , TestEvents , Reporter + , module Test.Spec.Config ) where import Prelude @@ -37,6 +36,7 @@ import Pipes.Core (Producer, Pipe, (//>)) import Pipes.Core (runEffect, runEffectRec) as P import Prim.TypeError (class Warn, Text) import Test.Spec (Item(..), Spec, SpecT, SpecTree, Tree(..), collect) +import Test.Spec.Config (Config, defaultConfig) import Test.Spec.Console as Console import Test.Spec.Result (Result(..)) import Test.Spec.Runner.Event (Event, Execution(..)) @@ -49,30 +49,6 @@ import Test.Spec.Tree (Path, PathItem(..), countTests, isAllParallelizable, pare foreign import exit :: Int -> Effect Unit -type Config = - { slow :: Milliseconds - -- ^ Threshold of time beyond which a test is considered "slow". - - , timeout :: Maybe Milliseconds - -- ^ An optional timeout, applied to each individual test. When omitted, tests - -- are allowed to run forever. - - , exit :: Boolean - -- ^ When `true`, the runner will exit the Node process after running tests. - -- If `false`, the runner will merely return test results. - - , failFast :: Boolean - -- ^ When `true`, first failed test stops the whole run. - } - -defaultConfig :: Config -defaultConfig = - { slow: Milliseconds 75.0 - , timeout: Just $ Milliseconds 2000.0 - , exit: true - , failFast: false - } - makeTimeout :: Milliseconds -> Aff Unit @@ -231,7 +207,7 @@ runSpec :: Array Reporter -> Spec Unit -> Aff Unit -runSpec = runSpec' defaultConfig +runSpec reporters spec = runSpec' defaultConfig reporters spec runSpec' :: Config diff --git a/test.dhall b/test.dhall index a1bb1a1..2243651 100644 --- a/test.dhall +++ b/test.dhall @@ -2,5 +2,15 @@ let config = ./spago.dhall in config // { sources = config.sources # [ "test/**/*.purs" ], - dependencies = config.dependencies # [ "console" ] + dependencies = config.dependencies # + [ "console" + , "node-buffer" + , "node-child-process" + , "node-fs" + , "node-fs-aff" + , "node-os" + , "node-process" + , "node-streams" + , "refs" + ] } diff --git a/test/Integration.purs b/test/Integration.purs new file mode 100644 index 0000000..60bd9b4 --- /dev/null +++ b/test/Integration.purs @@ -0,0 +1,143 @@ +module Test.Integration where + +import Prelude + +import Control.Monad.Trans.Class (lift) +import Data.Either (Either(..)) +import Data.Maybe (Maybe(..)) +import Data.String as S +import Data.String as Str +import Data.Traversable (for_, traverse_) +import Effect (Effect) +import Effect.Aff (Aff, makeAff, nonCanceler) +import Effect.Class (liftEffect) +import Effect.Class.Console (log) +import Effect.Ref as Ref +import Node.Buffer as Buffer +import Node.ChildProcess (defaultSpawnOptions) +import Node.ChildProcess as IO +import Node.ChildProcess as Proc +import Node.Encoding (Encoding(..)) +import Node.FS.Aff as FS +import Node.FS.Stats (isDirectory) +import Node.OS (tmpdir) +import Node.Process (cwd) +import Node.Stream as Stream +import Test.Spec (SpecT, afterAll, describe, focus, it) +import Test.Spec.Assertions (shouldEqual) + +-- | Reads the contents of `/integration-tests/cases` and turns each +-- | subdirectory into a test case. See `/integration-tests/cases/README` for +-- | more details. +integrationSpecs :: SpecT Aff Unit Aff Unit +integrationSpecs = focus do + { runFile, cleanupEnvironment } <- liftEffect $ prepareEnvironment { debug: false } + + afterAll (\_ -> cleanupEnvironment) $ + describe "Integration tests" do + cases <- lift $ FS.readdir "integration-tests/cases" + + for_ cases \testName -> do + let testDir = "integration-tests/cases/" <> testName + it testName do + program <- FS.readTextFile UTF8 $ testDir // "Main.purs" + goldenOutput <- FS.readTextFile UTF8 $ testDir // "output.txt" + actualOutput <- runFile program + Str.trim actualOutput `shouldEqual` Str.trim goldenOutput + +prepareEnvironment :: { debug :: Boolean } -> Effect { runFile :: String -> Aff String, cleanupEnvironment :: Aff Unit } +prepareEnvironment { debug } = + Ref.new Nothing <#> \envDirVar -> + { runFile: \program -> do + dir <- ensureEnvironmentInitialized envDirVar + ensureDirExists $ dir // "test" + FS.writeTextFile UTF8 (dir // "test/Main.purs") program + run' dir "npx" ["spago", "build"] + res <- run dir "npx" $ ["spago", "--quiet", "--no-psa", "test"] + pure $ + -- Removing ESC characters (which are used for colors), because + -- they're very inconvenient to include in the golden output files. + S.replaceAll (S.Pattern "\x1B") (S.Replacement "") res + + , cleanupEnvironment: + liftEffect (Ref.read envDirVar) >>= case _ of + Just dir | debug -> do + traceLog "Skipping environment cleanup due to debug=true flag" + traceLog $ "Environment at: " <> dir + Just dir -> + rmdirRec dir + Nothing -> + pure unit + } + where + traceLog + | debug = log + | otherwise = const $ pure unit + + ensureEnvironmentInitialized envDirVar = + liftEffect (Ref.read envDirVar) >>= case _ of + Just d -> + pure d + Nothing -> do + dir <- liftEffect tmpdir >>= \tmp -> FS.mkdtemp $ tmp // "purescript-spec-test-env" + liftEffect $ Ref.write (Just dir) envDirVar + traceLog $ "Preparing environment in: " <> dir + copyAllFiles { from: "integration-tests/env-template", to: dir } + patchRepoPath $ dir // "packages.dhall" + run' dir "npm" ["install", "purescript@0.15", "spago@0.21"] + pure dir + + patchRepoPath file = do + thisDir <- liftEffect cwd + content <- FS.readTextFile UTF8 file + let newContent = S.replaceAll (S.Pattern "SPEC_REPO_PATH") (S.Replacement thisDir) content + FS.writeTextFile UTF8 file newContent + + copyAllFiles { from, to } = do + ensureDirExists to + FS.readdir from >>= traverse_ \f -> do + stat <- FS.stat f + if isDirectory stat then + copyAllFiles { from: from // f, to: to // f } + else + FS.copyFile (from // f) (to // f) + + run' cwd cmd = void <<< run cwd cmd + + run cwd cmd args = do + traceLog $ "Running: " <> S.joinWith " " ([cmd] <> args) + makeAff \cb -> do + output <- Ref.new "" + let return = cb <<< Right =<< Ref.read output + + proc <- Proc.spawn cmd args defaultSpawnOptions { cwd = Just cwd, stdio = [Just IO.Ignore, Just IO.Pipe, Just IO.Pipe] } + + for_ [Proc.stdout, Proc.stderr] \pipe -> + Stream.onData (pipe proc) \buf -> do + str <- Buffer.toString UTF8 buf + void $ output # Ref.modify (_ <> str) + + Proc.onExit proc \_ -> return + Proc.onError proc \_ -> return + Proc.onDisconnect proc return + Proc.onClose proc \_ -> return + + pure nonCanceler + + ensureDirExists dir = + FS.access dir >>= case _ of + Just _err -> FS.mkdir dir + Nothing -> pure unit + + rmdirRec dir = do + stat <- FS.stat dir + if isDirectory stat then do + FS.readdir dir >>= traverse_ \f -> rmdirRec $ dir // f + FS.rmdir dir + else + FS.unlink dir + +pathConcat :: String -> String -> String +pathConcat a b = a <> "/" <> b + +infixl 5 pathConcat as // diff --git a/test/Main.purs b/test/Main.purs index abb582e..bcbdd8a 100644 --- a/test/Main.purs +++ b/test/Main.purs @@ -2,22 +2,31 @@ module Test.Main where import Prelude +import Data.Maybe (Maybe(..)) +import Data.Newtype (unwrap) import Effect (Effect) -import Effect.Aff (launchAff_) +import Effect.Aff (Milliseconds(..), launchAff_) +import Test.Integration (integrationSpecs) +import Test.Spec (hoistSpec) import Test.Spec.AssertionSpec (assertionSpec) import Test.Spec.HoistSpec (hoistSpecSpec) import Test.Spec.HookSpec (hookSpec) import Test.Spec.ParallelSpec (parallelSpec) import Test.Spec.Reporter (specReporter) import Test.Spec.Reporter.TeamCitySpec (teamcitySpec) -import Test.Spec.Runner (runSpec) +import Test.Spec.Runner (defaultConfig, runSpecT) import Test.Spec.RunnerSpec (runnerSpec) main :: Effect Unit -main = launchAff_ $ runSpec [specReporter] do - runnerSpec - assertionSpec - hookSpec - hoistSpecSpec - parallelSpec - teamcitySpec +main = launchAff_ $ void $ join $ + runSpecT defaultConfig { timeout = Just $ Milliseconds 60000.0 } [specReporter] do + pureSpecs + integrationSpecs + where + pureSpecs = hoistSpec (pure <<< unwrap) (\_ x -> x) do + runnerSpec + assertionSpec + hookSpec + hoistSpecSpec + parallelSpec + teamcitySpec