Merge pull request #435 from oughtinc/rachbach/oug-1101-make-ci-run-b…

…etter

.travis.yml

@@ -1,16 +1,19 @@
 sudo: false
 language: python
-python:
-- 3.6.9
-- 3.7.9
-- 3.8.5
+jobs:
+  include:
+    - python: 3.6.9
+      script: make test_skip_metaculus
+    - python: 3.7.9
+      script: make test
+    - python: 3.8.5
+      script: make test_skip_metaculus
 before_install:
-- pip install poetry
+  - pip install poetry
 install:
-- poetry install
+  - poetry install
 script:
-- make lint
-- make docs
-- make test
+  - make lint
+  - make docs
 after_success:
-- poetry run codecov
+  - poetry run codecov

Makefile

@@ -9,6 +9,9 @@ lint: FORCE  ## Run isort, flake8, mypy and black (in check mode)
 test: FORCE  ## Run pytest
 	poetry run python -m pytest --cov=ergo --ff --verbose -s --doctest-modules .
 
+test_skip_metaculus: FORCE  ## Run pytest, but skip the Metaculus tests to avoid overburdening the Metaculus API
+	poetry run python -m pytest --cov=ergo --ff --verbose -s --doctest-modules --ignore-glob='*test_metaculus.py' .
+
 xtest: FORCE  ## Run pytest in parallel mode using xdist
 	poetry run python -m pytest --cov=ergo --ff --verbose -s --doctest-modules -n auto .
 

ergo/platforms/metaculus/metaculus.py

@@ -1,40 +1,6 @@
 """
 This module lets you get question and prediction information from Metaculus
 and submit predictions, via the API (https://www.metaculus.com/api2/)
-
-**Example**
-
-In this example, we predict the admit rate for Harvard's class of 2029:
-
-https://www.metaculus.com/questions/3622
-
-We predict that the admit rate will be 20% higher than the current community prediction.
-
-.. doctest::
-    >>> import os
-    >>> import ergo
-    >>> import jax.numpy as np
-    >>> from numpyro.handlers import seed
-
-    >>> metaculus = ergo.Metaculus(
-    ...     api_domain="www"
-    ... )
-    >>> metaculus.login_via_username_and_password(
-    ...     username=os.getenv("METACULUS_USERNAME"),
-    ...     password=os.getenv("METACULUS_PASSWORD"),
-    ... )
-
-    >>> harvard_question = metaculus.get_question(3622)
-    >>> # harvard_question.show_community_prediction()
-
-    >>> community_prediction_samples = np.array(
-    ...     [harvard_question.sample_community() for _ in range(0, 5000)]
-    ... )
-    >>> my_prediction_samples = community_prediction_samples * 1.2
-
-    >>> # harvard_question.show_prediction(my_prediction_samples)
-    >>> harvard_question.submit_from_samples(my_prediction_samples)
-    <Response [202]>
 """
 from datetime import datetime
 import json

tests/conftest.py

@@ -11,6 +11,10 @@
 from ergo.distributions import Logistic, LogisticMixture, Truncate
 from ergo.scale import LogScale, Scale, TimeScale
 
+METACULUS_USERNAME = "oughttest"
+METACULUS_PASSWORD = "6vCo39Mz^rrb"
+METACULUS_USER_ID = "112420"
+
 
 def three_sd_scale(loc, s):
     sd = s * math.pi / math.sqrt(3)
@@ -149,9 +153,9 @@ def log_question_data():
 @pytest.fixture(scope="module")
 def metaculus():
     load_dotenv()
-    uname = cast(str, os.getenv("METACULUS_USERNAME"))
-    pwd = cast(str, os.getenv("METACULUS_PASSWORD"))
-    user_id_str = cast(str, os.getenv("METACULUS_USER_ID"))
+    uname = METACULUS_USERNAME
+    pwd = METACULUS_PASSWORD
+    user_id_str = METACULUS_USER_ID
     if None in [uname, pwd, user_id_str]:
         raise ValueError(
             ".env is missing METACULUS_USERNAME, METACULUS_PASSWORD, or METACULUS_USER_ID"

tests/test_metaculus.py

@@ -5,6 +5,7 @@
 import jax.numpy as np
 import pytest
 import requests
+import os
 
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -247,6 +248,10 @@ def test_sample_community_binary(metaculus_questions):
     assert bool(value) in (True, False)
 
 
+@pytest.mark.skipif(
+    "METACULUS_ORG_API_KEY" not in os.environ,
+    reason="Can only test with secret API keys",
+)
 def test_submit_binary_via_api_keys(metaculus_via_api_keys):
     question = metaculus_via_api_keys.get_question(3616)
     r = question.submit(0.55)