ENH: Continuous integration

.gitlab-ci.yml

@@ -0,0 +1,61 @@
+stages:
+  - check
+  - build
+  - test
+
+check_flake8:
+  stage: check
+  image: it4i/loombuilder:latest
+  script:
+    - flake8 python/loom/client
+    - flake8 python/loom/lore
+    - flake8 python/loom/wside
+    - flake8 tests/client
+
+build_job:
+  stage: build
+  image: it4i/loombuilder:latest
+  script:
+    - mkdir _build
+    - cd _build
+    - cmake ..
+    - make -j 2
+    - cd ../python
+    - sh generate.sh
+  artifacts:
+    paths:
+      - _build
+      - python
+    expire_in:
+      6h
+
+test_pytest:
+  stage: test
+  image: it4i/loombuilder:latest
+  script:
+    - cd tests/client
+    - py.test -x -v --timeout=20
+  artifacts:
+    when: on_failure
+    paths:
+      - tests/client/build
+    expire_in:
+      1d
+
+test_doc:
+  stage: test
+  image: it4i/loombuilder:latest
+  script:
+    - cd doc
+    - make html
+  artifacts:
+    paths:
+      - doc/build/html
+    expire_in:
+      1h
+
+test_catch:
+  stage: test
+  image: it4i/loombuilder:latest
+  script:
+    - _build/tests/cpp/cpp-test

python/loom/client/tasks.py

@@ -21,6 +21,7 @@ def cpus(n):
     r.add_resource("loom/resource/cpus", n)
     return r
 
+
 cpu1 = cpus(1)
 
 

python/loom/client/wait/__main__.py

@@ -8,6 +8,7 @@
 import time
 from ..client import Client
 
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Wait "
@@ -37,7 +38,6 @@ def parse_args():
     return args
 
 
-
 def main():
     args = parse_args()
     client = Client(args.server, args.port)

tests/client/loomenv.py

@@ -94,9 +94,9 @@ def start(self, workers_count, cpus=1):
         for i in range(workers_count):
             w = self.start_process("worker{}".format(i), worker_args, env)
             workers.append(w)
-        time.sleep(0.15)
+        time.sleep(0.25)
         if VALGRIND:
-            time.sleep(10)
+            time.sleep(5)
         assert not server.poll()
         assert not any(w.poll() for w in workers)
 
@@ -148,7 +148,7 @@ def set_trace(self, trace_path):
         self.client.set_trace(self.get_filename(trace_path))
 
     def independent_python(self, code):
-        return self.start_process("python",
+        return self.start_process("python3",
                                   (sys.executable, "-c", code),
                                   catch_io=False)
 

tests/client/test_cv.py

@@ -1,13 +1,19 @@
-from loomenv import loom_env, LOOM_TESTPROG, LOOM_TEST_DATA_DIR  # noqa
-import loom.client.tasks as tasks  # noqa
+from loomenv import loom_env, LOOM_TEST_DATA_DIR
+import loom.client.tasks as tasks
 
 import os
+import shutil
+import pytest
 
 IRIS_DATA = os.path.join(LOOM_TEST_DATA_DIR, "iris.data")
 
 loom_env  # silence flake8
 
+has_libsvm = shutil.which("svm-train") is not None \
+             and shutil.which("svm-predict") is not None
 
+
+@pytest.mark.skipif(not has_libsvm, reason="libsvm not installed")
 def test_cv_iris(loom_env):
         CHUNKS = 15
         CHUNK_SIZE = 150 // CHUNKS  # There are 150 irises
@@ -39,8 +45,6 @@ def test_cv_iris(loom_env):
             task.label = "svm-predict"
             predict.append(task)
 
-        #loom_env.make_dry_report(predict, "dry.report")
-
         loom_env.set_trace("mytrace")
         results = loom_env.submit(predict)
 

tests/client/test_data.py

@@ -130,7 +130,7 @@ def test_data_save(loom_env):
         loom_env.submit(b)
 
     with pytest.raises(client.TaskFailed):
-        b = tasks.save(a1, "/etc/xxx")
+        b = tasks.save(a1, "/")
         loom_env.submit(b)
 
     b1 = tasks.save(a1, loom_env.get_filename("file1"))

tests/client/test_py.py

@@ -286,5 +286,4 @@ def test_py_task_deserialization3(loom_env):
     loom_env.start(2)
     objs = tuple(tasks.py_value(str(i + 1000)) for i in range(100))
     x = tasks.array_make(objs)
-    result = loom_env.submit(x)
-
+    loom_env.submit(x)

tests/client/test_run.py

@@ -11,7 +11,7 @@
 
 
 def pytestprog(sleep, op="copy", stamp=False, file_out=None, file_in=None):
-    args = ["/usr/bin/python", LOOM_TESTPROG]
+    args = ["/usr/bin/python3", LOOM_TESTPROG]
     if stamp:
         args.append("--stamp")
     if file_in:
@@ -160,7 +160,8 @@ def test_run_chain(loom_env):
     const = b"ABC" * 10000000
     a = tasks.const(const)
 
-    b1 = tasks.run(pytestprog(0.01, file_out="test"), stdin=a, outputs=["test"])
+    b1 = tasks.run(pytestprog(0.01, file_out="test"),
+                   stdin=a, outputs=["test"])
     b2 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"),
                    inputs=[(b1, "in")], outputs=["test"])
     b3 = tasks.run(pytestprog(0.01, file_in="in", file_out="test"),

tests/client/testprog.py

@@ -24,7 +24,7 @@ def main():
 
 def run(args):
     if args.stamp:
-        print datetime.now()
+        print(datetime.now())
 
     op = args.operation
     if op != "empty":
@@ -49,7 +49,7 @@ def run(args):
     time.sleep(args.sleep)
 
     if args.stamp:
-        print datetime.now()
+        print(datetime.now())
 
 
 if __name__ == "__main__":