Make sure duplicates work better

.travis.yml

@@ -1,5 +1,6 @@
 language: python
 sudo: required
+dist: xenial
 notifications:
   email: false
 
@@ -12,12 +13,20 @@ branches:
     - master
 
 cache:
-  pip: true
-  apt: true
+  - pip: true
+  - apt: true
+  - directories:
+    - "$HOME/.pyenv_cache"
 
 matrix:
   include:
-    - python: "3.7-dev" # 3.7 development branch
+    - python: "3.7" # 3.7 development branch
+      env: PYENV_VERSION="3.5.5"
+    - python: "3.7" # 3.7 development branch
+      env: PYENV_VERSION="3.6.3"
+    - python: "3.7" # 3.7 development branch
+      env: PYENV_VERSION="3.7.0"
+
 
 addons:
   apt:
@@ -26,19 +35,24 @@ addons:
       - flex
 
 before_install:
+  - export PYENV_ROOT="$HOME/.custom_pyenv"
+  - mkdir "$PYENV_ROOT"
   - sudo apt-get install -qq pkg-config fuse
   - sudo modprobe fuse
   - sudo chmod 666 /dev/fuse
   - sudo chown root:$USER /etc/fuse.conf
 
 install:
   - pip install -U wheel
-  - pip install coveralls pytest-cov codecov
-  - pip install .
-  - which cgfs
   - pip install celery[redis]
   - git clone https://github.com/CodeGra-de/CodeGra.de.git backend
   - pip install -r backend/requirements.txt
+  - export PYTHON_NEW="$(which python)"
+  - wget https://github.com/praekeltfoundation/travis-pyenv/releases/download/0.4.0/setup-pyenv.sh
+  - source setup-pyenv.sh
+  - pip install coveralls pytest-cov codecov
+  - pip install .
+  - which cgfs
 
 before_script:
   - pwd
@@ -47,21 +61,22 @@ before_script:
   - echo -e '\n[Celery]\nbroker_url = redis://localhost:6379/0\n' >> backend/config.ini
   - mkdir -p backend/uploads
   - mkdir -p backend/mirror_uploads
-  - python --version
+  - $PYTHON_NEW --version
   - export PYTHONPATH="$PYTHONPATH:$(pwd)"
   - export DEBUG=true
   - psql -c 'create database travis_ci_test;' -U postgres
-  - export PYTHON=python
+  - export PYTHON=$PYTHON_NEW
   - cd backend
   - make db_upgrade
   - make test_data
-  - make start_dev_server > ../server.log 2>&1 &
+  - ( deactivate && source $(dirname $PYTHON_NEW)/activate && make start_dev_server > ../server.log 2>&1 & )
   - cd ..
   - sleep 6
 
 script:
   - make test
   - pip install mypy
+  - if [[ $PYENV_VERSION = "3.5.5" ]]; then > codegra_fs/cgfs_types.py; fi
   - mypy codegra_fs
   - pip install -r docs_requirements.txt || true
   - ( cd docs; make html )

codegra_fs/api_consumer.py

@@ -8,6 +8,11 @@
 import typing as t
 
 
+def json_loads(s: t.Union[bytes, str]) -> t.Dict[str, t.Any]:
+    if sys.version_info >= (3, 6):
+        return json.loads(s)
+    return json.loads(s if isinstance(s, str) else s.decode('utf8'))
+
 def print_usage() -> None:
     print(
         (
@@ -44,7 +49,7 @@ def is_file(s: socket.socket, file: str) -> int:
             }).encode('utf8')
         )
     )
-    if json.loads(recv(s))['ok']:
+    if json_loads(recv(s))['ok']:
         return 0
     else:
         return 2
@@ -63,7 +68,7 @@ def get_comments(s: socket.socket, file: str) -> int:
     )
     message = recv(s)
 
-    out = json.loads(message)
+    out = json_loads(message)
     if out['ok']:
         res = []
         for key, val in out['data'].items():
@@ -88,7 +93,7 @@ def delete_comment(s: socket.socket, file: str, line: int) -> int:
             ).encode('utf8')
         )
     )
-    if json.loads(recv(s))['ok']:
+    if json_loads(recv(s))['ok']:
         return 0
     else:
         return 2
@@ -107,7 +112,7 @@ def set_comment(s: socket.socket, file: str, line: int, message: str) -> int:
             ).encode('utf8')
         )
     )
-    if json.loads(recv(s))['ok']:
+    if json_loads(recv(s))['ok']:
         return 0
     else:
         return 2

codegra_fs/cgapi.py

@@ -4,16 +4,37 @@
 import typing as t
 import logging
 from enum import IntEnum
-from functools import partial
 from urllib.parse import quote
 
 import requests
 
 DEFAULT_CGAPI_BASE_URL = 'https://codegra.de/api/v1'
 
+T_CALL = t.TypeVar('T_CALL', bound=t.Callable)
+
 logger = logging.getLogger(__name__)
 
 
+def make_request_method(fun: T_CALL) -> T_CALL:
+    def meth(*args, **kwargs):
+        try:
+            return fun(*args, **kwargs, timeout=3)
+        except requests.exceptions.ConnectionError:
+            try:
+                sleep(1)
+                return fun(*args, **kwargs, timeout=3)
+            except requests.exceptions.ConnectionError as e:
+                raise CGAPIException(
+                    {
+                        'message': str(e),
+                        'description': str(e),
+                        'code': 500,
+                    }
+                )
+
+    return t.cast(T_CALL, meth)
+
+
 class APIRoutes():
     def __init__(self, base, owner='auto'):
         while base and base[-1] == '/':  # pragma: no cover
@@ -156,8 +177,8 @@ def __init__(
         self,
         username: str,
         password: str,
-        base: t.Optional[str]=None,
-        fixed: bool=False
+        base: t.Optional[str] = None,
+        fixed: bool = False
     ) -> None:
         owner = 'student' if fixed else 'auto'
         self.routes = APIRoutes(base or DEFAULT_CGAPI_BASE_URL, owner)
@@ -187,11 +208,11 @@ def __init__(
         self.s.headers = {
             'Authorization': 'Bearer ' + self.access_token,
         }
-        self.s.get = partial(self.s.get, timeout=3)  # type: ignore
-        self.s.patch = partial(self.s.patch, timeout=3)  # type: ignore
-        self.s.post = partial(self.s.post, timeout=3)  # type: ignore
-        self.s.delete = partial(self.s.delete, timeout=3)  # type: ignore
-        self.s.put = partial(self.s.put, timeout=3)  # type: ignore
+        self.s.get = make_request_method(self.s.get)  # type: ignore
+        self.s.patch = make_request_method(self.s.patch)  # type: ignore
+        self.s.post = make_request_method(self.s.post)  # type: ignore
+        self.s.delete = make_request_method(self.s.delete)  # type: ignore
+        self.s.put = make_request_method(self.s.put)  # type: ignore
 
     def _handle_response_error(self, request):
         if request.status_code >= 400:
@@ -350,8 +371,8 @@ def get_submission(self, submission_id):
     def set_submission(
         self,
         submission_id: int,
-        grade: t.Union[None, float, str]=None,
-        feedback: t.Optional[bytes]=None
+        grade: t.Union[None, float, str] = None,
+        feedback: t.Optional[bytes] = None
     ):
         url = self.routes.set_submission(submission_id)
         d = {}  # type: t.Dict[str, t.Union[bytes, float, None]]