test: marked evals, disable running by default

Author: ErikBjare

Makefile

@@ -23,6 +23,7 @@ test:
 		--cov=gptme --cov-report=xml --cov-report=term-missing --cov-report=html \
 		-n 8 \
 		$(if $(SLOW), --timeout 60, --timeout 5 -m "not slow") \
+		$(if $(EVAL), , -m "not eval") \
 		$(if $(PROFILE), --profile-svg) \
 		$(if $(HAS_PLAYWRIGHT), --cov-config=scripts/.coveragerc-playwright)
 

eval/test_eval.py

@@ -1,10 +1,11 @@
 import pytest
-from evals import tests
 
+from evals import tests
 from main import execute
 
 
 @pytest.mark.slow
+@pytest.mark.eval
 def test_eval(test):
     """
     This test will be run for each eval in the tests list.

pyproject.toml

@@ -84,6 +84,7 @@ ignore = ["E402", "E501"]
 [tool.pytest.ini_options]
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "eval: marks tests as evals (deselect with '-m \"not eval\"'), these may fail, and will not run in CI",
     "serial",
 ]