Merge pull request snabbco#52 from lukego/nix-update

Update nix code & import testsuite as subtree

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "submodules/raptorjit-testsuite"]
-	path = submodules/raptorjit-testsuite
-	url = https://github.com/raptorjit/raptorjit-testsuite

.travis.yml

@@ -1,12 +1,9 @@
 language: nix
 sudo: false
 env:
-  - jit=-O3
-  - jit=-O2
-  - jit=-O1
-  - jit=-joff
+  - test=test-O3
+  - test=test-O2
+  - test=test-O1
+  - test=test-nojit
 script:
-  - nix-build
-  - result/bin/raptorjit -e 'for i = 1, 1000 do end'
-  - git submodule init submodules/raptorjit-testsuite
-  - (cd submodules/raptorjit-testsuite/test; ../../../result/bin/raptorjit $jit test.lua +slow)
+  - nix-build -A $test

README.md

@@ -76,6 +76,10 @@ bootstrapping (see [default.nix](default.nix)). The recommended way to
 build RaptorJIT is with nix, which provides the dependencies
 automatically, but you can build manually if you prefer.
 
+Building with nix will be slow the first time due to downloading
+toolchains and related dependencies. This is all cached for future
+builds.
+
 #### Build with nix
 
 Install nix:
@@ -84,12 +88,18 @@ Install nix:
 $ curl https://nixos.org/nix/install | sh
 ```
 
-Build in batch-mode (option 1):
+Build in batch-mode and run the test suite (option 1a):
 
 ```shell
 $ nix-build    # produces result/bin/raptorjit
 ```
 
+Build in batch-mode without the test suite (option 1b):
+
+```shell
+$ nix-build -A raptorjit
+```
+
 Build interactively (option 2):
 
 ```shell

default.nix

@@ -7,24 +7,22 @@
 #
 # See README.md for usage instructions.
 
-{ pkgs ? (import <nixpkgs> {}) # Use default nix distro (for now...)
-, source ? ./.
-, version ? "dev"
-}:
+{ pkgs ? (import ./pkgs.nix) {}
+, source ? pkgs.lib.cleanSource ./.
+, version ? "dev" }:
 
-with pkgs;
-with clangStdenv;            # Clang instead of GCC
+let
+  callPackage = (pkgs.lib.callPackageWith { inherit pkgs; inherit source; inherit version; });
+  raptorjit = (callPackage ./raptorjit.nix {});
+  test = name: args: (callPackage ./test.nix { inherit raptorjit; inherit name; inherit args; });
+in
 
-mkDerivation rec {
-  name = "raptorjit-${version}";
-  inherit version;
-  src = lib.cleanSource source;
-  buildInputs = [ luajit ];  # LuaJIT to bootstrap DynASM
-  installPhase = ''
-    mkdir -p $out/bin
-    cp src/luajit $out/bin/raptorjit
-  '';
-
-  enableParallelBuilding = true;  # Do 'make -j'
+# Build RaptorJIT and run mulitple test suites.
+{
+  raptorjit  = raptorjit;
+  test-O3    = test "O3"    "-O3";
+  test-O2    = test "O2"    "-O2";
+  test-O1    = test "O1"    "-O1";
+  test-nojit = test "nojit" "-joff";
 }
 

pkgs.nix

@@ -0,0 +1 @@
+import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/6a0155d2b7cb10aef1c63b654a2b172d78fd89b4.tar.gz)

raptorjit.nix

@@ -0,0 +1,20 @@
+# raptorjit.nix - compile RaptorJIT with reference toolchain
+
+{ pkgs, source, version }:
+
+with pkgs;
+with llvmPackages_4.stdenv;  # Use clang 4.0
+
+mkDerivation rec {
+  name = "raptorjit-${version}";
+  inherit version;
+  src = source;
+  buildInputs = [ luajit ];  # LuaJIT to bootstrap DynASM
+  installPhase = ''
+    mkdir -p $out/bin
+    cp src/luajit $out/bin/raptorjit
+  '';
+
+  enableParallelBuilding = true;  # Do 'make -j'
+}
+

test.nix

@@ -0,0 +1,19 @@
+{ pkgs, raptorjit, source, name, args }:
+
+pkgs.stdenv.mkDerivation {
+  name = "test-${name}";
+  src = source;
+  buildInputs = [ raptorjit ];
+  phases = "unpackPhase buildPhase";
+  buildPhase = ''
+    mkdir $out
+    cd testsuite/test
+    echo "Running testsuite with ${args} and output to $out/log.txt"
+    raptorjit ${args} test.lua 2>&1 > $out/log.txt
+    result=$?
+    echo -n "*** TEST RESULTS (${args}): "
+    tail -1 $out/log.txt
+    exit $result
+  '';
+}
+

testsuite/README

@@ -0,0 +1,77 @@
+********************************************
+** THIS IS NOT THE TEST SUITE FOR LUAJIT! **
+********************************************
+
+In fact it doesn't even have the steps to build it or run it,
+so please don't complain.
+
+This repo is a place to collect and cleanup tests for LuaJIT.
+They should eventually be merged into the main LuaJIT repo.
+
+It's definitely not in the best state and needs a serious
+cleanup effort. Sorry.
+
+
+Many issues need to be resolved before the merge can be performed:
+
+- Choose a portable test runner
+  Requirement: very few dependencies, possibly Lua/Shell only
+
+- Minimal test runner library, wherever assert() is not enough
+
+- Debugging test failures is a lot simpler, when individual tests can still
+  be run from the LuaJIT command line without any big dependencies
+
+- Define consistent grouping of all tests
+
+- Define consistent naming of all tests
+
+- Split everything into a lot of tiny tests
+
+- Reduce time taken to run the test suite
+  Separate tiers, parallelized testing
+
+- Some tests can only run under certain configurations (e.g. FFI)
+
+- Some tests need a clean slate to give reproducible results
+  Most others should be run from the same state for performance resons
+
+- Hard to check that the JIT compiler actually generates the intended code
+  Maybe use a test matching variant of the jit.dump module
+
+- Portability concerns
+
+- Avoiding undefined behavior in tests or ignoring it
+
+- Matrix of architectures + configuration options that need testing
+
+- Merge tests from other sources, e.g. the various Lua test suites.
+
+- Tests should go into the LuaJIT git repo, but in separate tarballs
+  for the releases
+
+
+There are some benchmarks, too:
+
+- Some of the benchmarks can be used as tests (with low scaling)
+  by checksumming their output and comparing against known good results
+
+- Most benchmarks need different scalings to be useful for comparison
+  on all architectures
+
+
+Note from Mike Pall:
+
+I've removed all tests of undeterminable origin or that weren't explicitly
+contributed with the intention of being part of a public test suite.
+
+I hereby put all Lua/LuaJIT tests and benchmarks that I wrote under the
+public domain. I've removed any copyright headers.
+
+If I've forgotten an attribution or you want your contributed test to be
+removed, please open an issue.
+
+There are some benchmarks that bear other copyrights, probably public
+domain, BSD or MIT licensed. If the status cannot be determined, they
+need to be replaced or removed before merging with the LuaJIT repo.
+

testsuite/bench/PARAM_arm.txt

@@ -0,0 +1,29 @@
+array3d 200
+binary-trees 13
+chameneos 1e6
+coroutine-ring 3e6
+euler14-bit 5e6
+fannkuch 10
+fasta 2e6
+k-nucleotide 5e5 FASTA_500000
+life
+mandelbrot 2000
+mandelbrot-bit 2000
+md5 5000
+nbody 1e6
+nsieve 9
+nsieve-bit 9
+nsieve-bit-fp 9
+partialsums 2e6
+pidigits-nogmp 2000
+ray 4
+recursive-ack 9
+recursive-fib 37
+revcomp 1e6 FASTA_1000000
+scimark-fft 2000
+scimark-lu 300
+scimark-sor 5000
+scimark-sparse 5e3
+series 1500
+spectral-norm 1000
+sum-file 1000 SUMCOL_1000

testsuite/bench/PARAM_mips.txt

@@ -0,0 +1,29 @@
+array3d 50
+binary-trees 10
+chameneos 5e4
+coroutine-ring 2e5
+euler14-bit 2e4
+fannkuch 8
+fasta 2e4
+k-nucleotide 1e4 FASTA_10000
+life
+mandelbrot 150
+mandelbrot-bit 150
+md5 10
+nbody 1e4
+nsieve 4
+nsieve-bit 4
+nsieve-bit-fp 2
+partialsums 5e4
+pidigits-nogmp 150
+ray 2
+recursive-ack 7
+recursive-fib 29
+revcomp 5e4 FASTA_50000
+scimark-fft 20
+scimark-lu 3
+scimark-sor 40
+scimark-sparse 100
+series 50
+spectral-norm 100
+sum-file 100 SUMCOL_100

testsuite/bench/PARAM_ppc.txt

@@ -0,0 +1,29 @@
+array3d 200
+binary-trees 13
+chameneos 1e6
+coroutine-ring 4e6
+euler14-bit 1e6
+fannkuch 9
+fasta 5e5
+k-nucleotide 1e5 FASTA_100000
+life
+mandelbrot 800
+mandelbrot-bit 800
+md5 500
+nbody 1e5
+nsieve 8
+nsieve-bit 8
+nsieve-bit-fp 8
+partialsums 5e5
+pidigits-nogmp 800
+ray 5
+recursive-ack 9
+recursive-fib 34
+revcomp 1e6 FASTA_1000000
+scimark-fft 500
+scimark-lu 100
+scimark-sor 1000
+scimark-sparse 3000
+series 1000
+spectral-norm 200
+sum-file 1000 SUMCOL_1000

testsuite/bench/PARAM_x86.txt

@@ -0,0 +1,29 @@
+array3d 300
+binary-trees 16
+chameneos 1e7
+coroutine-ring 2e7
+euler14-bit 2e7
+fannkuch 11
+fasta 25e6
+k-nucleotide 5e6 FASTA_5000000
+life
+mandelbrot 5000
+mandelbrot-bit 5000
+md5 20000
+nbody 5e6
+nsieve 12
+nsieve-bit 12
+nsieve-bit-fp 12
+partialsums 1e7
+pidigits-nogmp 5000
+ray 9
+recursive-ack 10
+recursive-fib 40
+revcomp 5e6 FASTA_5000000
+scimark-fft 50000
+scimark-lu 5000
+scimark-sor 50000
+scimark-sparse 15e4
+series 10000
+spectral-norm 3000
+sum-file 5000 SUMCOL_5000

testsuite/bench/PARAM_x86_CI.txt

@@ -0,0 +1,27 @@
+array3d 300
+binary-trees 16
+chameneos 1e7
+coroutine-ring 2e7
+euler14-bit 2e7
+fannkuch 11
+fasta 5e6
+life
+mandelbrot 5000
+mandelbrot-bit 5000
+md5 20000
+nbody 5e6
+nsieve 12
+nsieve-bit 12
+nsieve-bit-fp 12
+partialsums 1e7
+pidigits-nogmp 5000
+ray 9
+recursive-ack 10
+recursive-fib 40
+scimark-fft 50000
+scimark-lu 5000
+scimark-sor 50000
+scimark-sparse 15e4
+series 10000
+spectral-norm 3000
+roulette