Add pre commit checks + CI (commaai#545)

* Add pre commit checks

* untested but should work

* fix typo in dockerfile

* add sqlite

* add pylintrc to ignore usb1 members

* Typo in stage name

* Move to requirements

.github/workflows/test.yaml

@@ -123,7 +123,6 @@ jobs:
       - uses: actions/checkout@v2
       - name: Build Docker image
         run: eval "$BUILD"
-      - name: Run linters
+      - name: Run static analysis
         run: |
-          $RUN "cd /tmp/openpilot/panda/tests/linter_python/ && ./flake8_panda.sh"
-          $RUN "cd /tmp/openpilot/panda/tests/linter_python/ && ./pylint_panda.sh"
+          $RUN "cd /tmp/openpilot/panda && git init && git add -A && MYPYPATH=/tmp/openpilot pre-commit run --all"

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: master
+    hooks:
+    -   id: check-ast
+    -   id: check-json
+    -   id: check-xml
+    -   id: check-yaml
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: master
+    hooks:
+    -   id: mypy
+        exclude: '^(tests/automated)/'
+-   repo: https://github.com/PyCQA/flake8
+    rev: master
+    hooks:
+    -   id: flake8
+        exclude: '^(tests/automated)/'
+        args:
+        - --select=F
+-   repo: local
+    hooks:
+    -   id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        exclude: '^(tests/automated)/'
+        args:
+        - --disable=R,C,W

.pylintrc

@@ -0,0 +1,2 @@
+[MASTER]
+generated-members=usb1.*

Dockerfile.panda

@@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libffi-dev \
     libtool \
     libssl-dev \
+    libsqlite3-dev \
     libusb-1.0-0 \
     libzmq3-dev \
     locales \
@@ -62,3 +63,4 @@ RUN cd /tmp/openpilot && \
     pip install --no-cache-dir -r tools/requirements.txt
 
 COPY . /tmp/openpilot/panda
+RUN rm -rf /tmp/openpilot/panda/.git

__init__.py

@@ -1 +1,3 @@
-from .python import Panda, PandaWifiStreaming, PandaDFU, ESPROM, CesantaFlasher, flash_release, BASEDIR, ensure_st_up_to_date, build_st, PandaSerial  # noqa: F401
+# flake8: noqa
+# pylint: skip-file
+from .python import Panda, PandaWifiStreaming, PandaDFU, ESPROM, CesantaFlasher, flash_release, BASEDIR, ensure_st_up_to_date, build_st, PandaSerial

python/__init__.py

@@ -10,12 +10,13 @@
 import traceback
 import subprocess
 import sys
-from .dfu import PandaDFU
-from .esptool import ESPROM, CesantaFlasher  # noqa: F401
-from .flash_release import flash_release  # noqa: F401
-from .update import ensure_st_up_to_date  # noqa: F401
-from .serial import PandaSerial  # noqa: F401
-from .isotp import isotp_send, isotp_recv
+from .dfu import PandaDFU  # pylint: disable=import-error
+from .esptool import ESPROM, CesantaFlasher  # noqa pylint: disable=import-error
+from .flash_release import flash_release  # noqa pylint: disable=import-error
+from .update import ensure_st_up_to_date  # noqa pylint: disable=import-error
+from .serial import PandaSerial  # noqa pylint: disable=import-error
+from .isotp import isotp_send, isotp_recv  # pylint: disable=import-error
+
 
 __version__ = '0.0.9'
 

python/uds.py

@@ -416,7 +416,7 @@ def _isotp_rx_next(self, rx_data: bytes) -> None:
       self.rx_idx = 0
       self.rx_done = False
       if self.debug: print(f"ISO-TP: RX - first frame - idx={self.rx_idx} done={self.rx_done}")
-      if self.debug: print(f"ISO-TP: TX - flow control continue")
+      if self.debug: print("ISO-TP: TX - flow control continue")
       # send flow control message (send all bytes)
       msg = b"\x30\x00\x00".ljust(self.max_len, b"\x00")
       self._can_client.send([msg])

requirements.txt

@@ -7,6 +7,7 @@ nose
 parameterized
 requests
 flake8==3.7.9
-pylint==2.4.3
 cffi==1.11.4
 crcmod
+pre-commit==2.4.0
+pylint==2.5.2

tests/all_wifi_test.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import requests
 import json
-from .automated.helpers import _connect_wifi
+from .automated.helpers import _connect_wifi  # pylint: disable=import-error
 from panda import Panda
 from nose.tools import assert_equal
 
@@ -25,5 +25,3 @@
     assert_equal(str(dongle_id), wifi_dongle_id)
     assert_equal(latest_version, st_version)
     assert_equal(latest_version, esp_version)
-
-

tests/automated/helpers.py

@@ -45,13 +45,13 @@ def init_panda_serials():
     param(panda_type=Panda.HW_TYPE_UNO)
   ])
 test_all_pandas = parameterized(
-    list(map(lambda x: x[0], _panda_serials))
+    list(map(lambda x: x[0], _panda_serials))  # type: ignore
   )
 test_all_gen2_pandas = parameterized(
-    list(map(lambda x: x[0], filter(lambda x: x[1] in GEN2_HW_TYPES, _panda_serials)))
+    list(map(lambda x: x[0], filter(lambda x: x[1] in GEN2_HW_TYPES, _panda_serials)))  # type: ignore
   )
 test_all_gps_pandas = parameterized(
-    list(map(lambda x: x[0], filter(lambda x: x[1] in GPS_HW_TYPES, _panda_serials)))
+    list(map(lambda x: x[0], filter(lambda x: x[1] in GPS_HW_TYPES, _panda_serials)))  # type: ignore
   )
 test_white_and_grey = parameterized([
     param(panda_type=Panda.HW_TYPE_WHITE_PANDA),
@@ -202,7 +202,7 @@ def wrapper(panda_serials=None, **kwargs):
     finally:
       # Close all connections
       for panda in pandas:
-        panda.close()        
+        panda.close()
   return wrapper
 
 def clear_can_buffers(panda):

tests/bulk_write_test.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
-import time
 import threading
+import time
+from typing import Any, List
 
 from panda import Panda
 
@@ -18,18 +19,18 @@ def flood_tx(panda):
   serials = Panda.list()
   if len(serials) != 2:
     raise Exception("Connect two pandas to perform this test!")
-  
+
   sender = Panda(serials[0])
   receiver = Panda(serials[1])
-  
+
   sender.set_safety_mode(Panda.SAFETY_ALLOUTPUT)
   receiver.set_safety_mode(Panda.SAFETY_ALLOUTPUT)
-  
+
   # Start transmisson
   threading.Thread(target=flood_tx, args=(sender,)).start()
 
   # Receive as much as we can in a few second time period
-  rx = []
+  rx: List[Any] = []
   old_len = 0
   start_time = time.time()
   while time.time() - start_time < 2 or len(rx) > old_len:

tests/debug_console.py

@@ -28,7 +28,7 @@
 
       if os.getenv("BAUD") is not None:
         for panda in pandas:
-          panda.set_uart_baud(port_number, int(os.getenv("BAUD")))
+          panda.set_uart_baud(port_number, int(os.getenv("BAUD"))) # type: ignore
 
       while True:
         for i, panda in enumerate(pandas):

tests/elm_car_simulator.py

@@ -299,10 +299,10 @@ def _process_obd(self, mode, pid):
 
 if __name__ == "__main__":
     serial = os.getenv("SERIAL") if os.getenv("SERIAL") else None
-    kbaud = int(os.getenv("CANKBAUD")) if os.getenv("CANKBAUD") else 500
-    bitwidth = int(os.getenv("CANBITWIDTH")) if os.getenv("CANBITWIDTH") else 0
-    canenable = bool(int(os.getenv("CANENABLE"))) if os.getenv("CANENABLE") else True
-    linenable = bool(int(os.getenv("LINENABLE"))) if os.getenv("LINENABLE") else True
+    kbaud = int(os.getenv("CANKBAUD")) if os.getenv("CANKBAUD") else 500  # type: ignore
+    bitwidth = int(os.getenv("CANBITWIDTH")) if os.getenv("CANBITWIDTH") else 0  # type: ignore
+    canenable = bool(int(os.getenv("CANENABLE"))) if os.getenv("CANENABLE") else True  # type: ignore
+    linenable = bool(int(os.getenv("LINENABLE"))) if os.getenv("LINENABLE") else True  # type: ignore
     sim = ELMCarSimulator(serial, can_kbaud=kbaud, can=canenable, lin=linenable)
     if(bitwidth == 0):
         sim.can_mode_11b_29b()