Update to T.O.P v0.9.4 DEV (NN Torque Control)

C2 version with NN Torque Controller
Synced with legacypilot master 2023.07.17 commits.

T.O.P function update
．New Vision Turn Control

Support list：
AUDI A3 3RD GEN
BUICK LACROSSE 2017
CHEVROLET BOLT EUV 2022
CHEVROLET BOLT EV NO ACC
CHEVROLET SILVERADO 1500 2020
CHEVROLET SUBURBAN PREMIER 2019
CHEVROLET TRAILBLAZER 2021
CHEVROLET VOLT PREMIER 2017
CHRYSLER PACIFICA HYBRID 2019
FORD MAVERICK 1ST GEN
GENESIS GV70 1ST GEN
GMC ACADIA DENALI 2018
HONDA CIVIC 2022
HONDA RIDGELINE 2017
HYUNDAI IONIQ 5 2022
HYUNDAI PALISADE 2020
HYUNDAI SONATA 2020
KIA EV6 2022
KIA SPORTAGE 5TH GEN
LEXUS ES 2019
MAZDA CX-9
RAM 1500 5TH GEN
RAM HD 5TH GEN
SKODA KAROQ 1ST GEN
SKODA KODIAQ 1ST GEN
SUBARU LEGACY 7TH GEN
SUBARU OUTBACK 6TH GEN
TOYOTA CAMRY HYBRID 2021
TOYOTA COROLLA HYBRID TSS2 2019
TOYOTA COROLLA TSS2 2019
TOYOTA PRIUS 2017
TOYOTA PRIUS TSS2 2021
TOYOTA PRIUS v 2017
TOYOTA RAV4 HYBRID 2019
VOLKSWAGEN GOLF 7TH GEN
VOLKSWAGEN JETTA 7TH GEN
VOLKSWAGEN PASSAT NMS
VOLKSWAGEN TIGUAN 2ND GEN

.github/workflows/selfdrive_tests.yaml

@@ -188,6 +188,7 @@ jobs:
   static_analysis:
     name: static analysis
     runs-on: ubuntu-20.04
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
       with:
@@ -200,6 +201,7 @@ jobs:
   valgrind:
     name: valgrind
     runs-on: ubuntu-20.04
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
       with:
@@ -217,6 +219,7 @@ jobs:
   unit_tests:
     name: unit tests
     runs-on: ubuntu-20.04
+    timeout-minutes: 50
     steps:
     - uses: actions/checkout@v3
       with:
@@ -242,6 +245,7 @@ jobs:
                         $UNIT_TEST selfdrive/athena && \
                         $UNIT_TEST selfdrive/thermald && \
                         $UNIT_TEST system/hardware/tici && \
+                        $UNIT_TEST selfdrive/mapd && \
                         $UNIT_TEST tools/lib/tests && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
@@ -259,47 +263,48 @@ jobs:
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
-  process_replay:
-    name: process replay
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - uses: ./.github/workflows/setup
-    - name: Cache test routes
-      id: dependency-cache
-      uses: actions/cache@v3
-      with:
-        path: /tmp/comma_download_cache
-        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Run replay
-      run: |
-        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
-                        coverage xml"
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v2
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload reference logs
-      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
-      run: |
-        ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v3
+  #process_replay:
+  #  name: process replay
+  #  runs-on: ubuntu-20.04
+  #  steps:
+  #  - uses: actions/checkout@v3
+  #    with:
+  #      submodules: true
+  #  - uses: ./.github/workflows/setup
+  #  - name: Cache test routes
+  #    id: dependency-cache
+  #    uses: actions/cache@v3
+  #    with:
+  #      path: /tmp/comma_download_cache
+  #      key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
+  #  - name: Build openpilot
+  #    run: |
+  #      ${{ env.RUN }} "scons -j$(nproc)"
+  #  - name: Run replay
+  #    run: |
+  #      ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+  #                      coverage xml"
+  #  - name: Print diff
+  #    id: print-diff
+  #    if: always()
+  #    run: cat selfdrive/test/process_replay/diff.txt
+  #  - uses: actions/upload-artifact@v2
+  #    if: always()
+  #    continue-on-error: true
+  #    with:
+  #      name: process_replay_diff.txt
+  #      path: selfdrive/test/process_replay/diff.txt
+  #  - name: Upload reference logs
+  #    if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+  #    run: |
+  #      ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+  #  - name: "Upload coverage to Codecov"
+  #    uses: codecov/codecov-action@v3
 
   test_modeld:
     name: model tests
     runs-on: ubuntu-20.04
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
       with:
@@ -325,6 +330,7 @@ jobs:
   test_longitudinal:
     name: longitudinal
     runs-on: ubuntu-20.04
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v3
       with:
@@ -351,6 +357,7 @@ jobs:
   test_cars:
     name: cars
     runs-on: ubuntu-20.04
+    timeout-minutes: 40
     strategy:
       fail-fast: false
       matrix:

.gitignore

@@ -32,7 +32,6 @@ a.out
 *.class
 *.pyxbldc
 *.vcd
-*.qm
 config.json
 clcache
 compile_commands.json

.gitmodules

@@ -1,18 +1,15 @@
 [submodule "panda"]
   path = panda
-  url = ../../dragonpilot-community/panda.git
-  branch = legacy
+  url = ../../CT921/panda.git
 [submodule "opendbc"]
   path = opendbc
-  url = ../../dragonpilot-community/opendbc.git
-  branch = legacy
+  url = ../../CT921/opendbc.git
 [submodule "laika_repo"]
   path = laika_repo
   url = ../../commaai/laika.git
 [submodule "cereal"]
   path = cereal
-  url = ../../dragonpilot-community/cereal.git
-  branch = legacy
+  url = ../../CT921/cereal.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = ../../commaai/rednose.git

.pre-commit-config.yaml

@@ -15,12 +15,12 @@ repos:
     -   id: check-merge-conflict
     -   id: check-symlinks
     -   id: check-added-large-files
-        args: ['--maxkb=100']
+        args: ['--maxkb=420']
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:
     -   id: codespell
-        exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
+        exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)|(selfdrive/mapd/test/.*.xml)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup

common/numpy_fast.py

@@ -17,3 +17,11 @@ def get_interp(xv):
 
 def mean(x):
   return sum(x) / len(x)
+
+def sign(x):
+  if x > 0.0:
+    return 1.0
+  elif x < 0.0:
+    return -1.0
+  else:
+    return 0.0

common/params.cc

@@ -204,9 +204,37 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"Version", PERSISTENT},
     {"VisionRadarToggle", PERSISTENT},
 
+    // TOP
+    {"CarModel", PERSISTENT},
+    {"dp_atl", PERSISTENT},
+    {"dp_jetson", PERSISTENT},
+    {"DrivingPersonalitiesUIWheel", PERSISTENT},
+    {"dynamic_lane", PERSISTENT},
+    {"e2e_link", PERSISTENT},
+    {"EnableTorqueController", PERSISTENT},
+    {"GpxDeleteAfterUpload", PERSISTENT},
+    {"GpxDeleteIfUploaded", PERSISTENT},
+    {"LastGPSPosition", PERSISTENT},
+    {"LateralAllowed", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"LiveTorque", PERSISTENT},
+    {"NudgelessLaneChange", PERSISTENT},
+    {"OnroadScreenOff", PERSISTENT},
+    {"PrimeAd", PERSISTENT},
+    {"ReverseAccChange", PERSISTENT},
+    {"TimSignals", PERSISTENT},
+    {"TogglesUpdated", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"topsng", PERSISTENT},
+    {"toyotaautolock", PERSISTENT},
+    {"toyotaautounlock", PERSISTENT},
+    {"toyota_bsm", PERSISTENT},
+    {"TurnVisionControl", PERSISTENT},
+    {"QuietDrive", PERSISTENT},
     {"dp_no_gps_ctrl", PERSISTENT},
     {"dp_no_fan_ctrl", PERSISTENT},
+    {"dp_mapd", PERSISTENT},
     {"dp_logging", PERSISTENT},
+    {"dp_fileserv", PERSISTENT},
+    {"dp_otisserv", PERSISTENT},
 };
 
 } // namespace

common/params.h

@@ -32,6 +32,10 @@ class Params {
   inline bool getBool(const std::string &key, bool block = false) {
     return get(key, block) == "1";
   }
+  inline int getInt(const std::string &key, bool block = false) {
+    std::string value = get(key, block);
+    return value.empty() ? 0 : std::stoi(value);
+  }
   std::map<std::string, std::string> readAll();
 
   // helpers for writing values
@@ -42,6 +46,9 @@ class Params {
   inline int putBool(const std::string &key, bool val) {
     return put(key.c_str(), val ? "1" : "0", 1);
   }
+  inline int putInt(const std::string &key, int val) {
+    return put(key.c_str(), std::to_string(val).c_str(), std::to_string(val).size());
+  }
 
 private:
   std::string params_path;

common/params_pyx.pyx

@@ -17,9 +17,11 @@ cdef extern from "common/params.h":
     c_Params(string) nogil
     string get(string, bool) nogil
     bool getBool(string, bool) nogil
+    int getInt(string, bool) nogil
     int remove(string) nogil
     int put(string, string) nogil
     int putBool(string, bool) nogil
+    int putInt(string, int) nogil
     bool checkKey(string) nogil
     string getParamPath(string) nogil
     void clearAll(ParamKeyType)
@@ -75,6 +77,13 @@ cdef class Params:
       r = self.p.getBool(k, block)
     return r
 
+  def get_int(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef int r
+    with nogil:
+      r = self.p.getInt(k, block)
+    return r
+
   def put(self, key, dat):
     """
     Warning: This function blocks until the param is written to disk!
@@ -92,6 +101,11 @@ cdef class Params:
     with nogil:
       self.p.putBool(k, val)
 
+  def put_int(self, key, int val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putInt(k, val)
+
   def remove(self, key):
     cdef string k = self.check_key(key)
     with nogil:
@@ -109,3 +123,6 @@ def put_nonblocking(key, val, d=""):
 
 def put_bool_nonblocking(key, bool val, d=""):
   threading.Thread(target=lambda: Params(d).put_bool(key, val)).start()
+
+def put_int_nonblocking(key, int val, d=""):
+  threading.Thread(target=lambda: Params(d).put_int(key, val)).start()

force_car_recognition.py

@@ -0,0 +1,8 @@
+filenames = ['/data/openpilot/selfdrive/car/top_tmp/HondaCars', '/data/openpilot/selfdrive/car/top_tmp/HyundaiCars', '/data/openpilot/selfdrive/car/top_tmp/SubaruCars',
+             '/data/openpilot/selfdrive/car/top_tmp/ToyotaCars', '/data/openpilot/selfdrive/car/top_tmp/VolkswagenCars']
+
+with open('/data/openpilot/selfdrive/car/top_tmp/Cars', 'w') as outfile:
+  for names in filenames:
+    with open(names) as infile:
+      outfile.write(infile.read())
+    outfile.write("\n")

launch_chffrplus.sh

@@ -125,6 +125,17 @@ function two_init {
   LIB_PATH="/data/openpilot/system/hardware/eon/libs"
   PY_LIB_DEST="/system/comma/usr/lib/python3.8/site-packages"
   mount -o remount,rw /system
+  # mapd
+  MODULE="opspline"
+  if [ ! -d "$PY_LIB_DEST/$MODULE" ]; then
+    echo "Installing $MODULE..."
+    tar -zxvf "$LIB_PATH/$MODULE.tar.gz" -C "$PY_LIB_DEST/"
+  fi
+  MODULE="overpy"
+  if [ ! -d "$PY_LIB_DEST/$MODULE" ]; then
+    echo "Installing $MODULE..."
+    tar -zxvf "$LIB_PATH/$MODULE.tar.gz" -C "$PY_LIB_DEST/"
+  fi
   # laika
   MODULE="hatanaka"
   if [ ! -d "$PY_LIB_DEST/$MODULE" ]; then
@@ -162,6 +173,18 @@ function two_init {
   fi
   mount -o remount,r /system
 
+  # osm server
+  if [ -f /data/params/d/dp_mapd ]; then
+    dp_mapd=`cat /data/params/d/dp_mapd`
+    if [ $dp_mapd == "1" ]; then
+      MODULE="osm-3s_v0.7.56"
+      if [ ! -d /data/media/0/osm/ ]; then
+        tar -vxf "/data/openpilot/system/hardware/eon/libs/$MODULE.tar.xz" -C /data/media/0/
+        mv "/data/media/0/$MODULE" /data/media/0/osm
+      fi
+    fi
+  fi
+
   # Check for NEOS update
   if [ -f /LEECO ] && [ $(< /VERSION) != "$REQUIRED_NEOS_VERSION" ]; then
     echo "Installing NEOS update"
@@ -190,11 +213,6 @@ function two_init {
 }
 
 function agnos_init {
-  # wait longer for weston to come up
-  if [ -f "$BASEDIR/prebuilt" ]; then
-    sleep 3
-  fi
-
   # TODO: move this to agnos
   sudo rm -f /data/etc/NetworkManager/system-connections/*.nmmeta
 
@@ -268,8 +286,17 @@ function launch {
   # write tmux scrollback to a file
   tmux capture-pane -pq -S-1000 > /tmp/launch_log
 
+  python ./selfdrive/car/honda/values.py > /data/openpilot/selfdrive/car/top_tmp/HondaCars
+  python ./selfdrive/car/hyundai/values.py > /data/openpilot/selfdrive/car/top_tmp/HyundaiCars
+  python ./selfdrive/car/subaru/values.py > /data/openpilot/selfdrive/car/top_tmp/SubaruCars
+  python ./selfdrive/car/toyota/values.py > /data/openpilot/selfdrive/car/top_tmp/ToyotaCars
+  python ./selfdrive/car/volkswagen/values.py > /data/openpilot/selfdrive/car/top_tmp/VolkswagenCars
+
+  python ./force_car_recognition.py
+
   # start manager
   cd selfdrive/manager
+  chmod 775 manager.py
   ./build.py && ./manager.py
 
   # if broken, keep on screen error