test_models: add fuzzy tx test

.github/workflows/selfdrive_tests.yaml

@@ -222,6 +222,13 @@ jobs:
       if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
         ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
+    - name: Run model replay with ONNX
+      timeout-minutes: 4
+      run: |
+        ${{ env.RUN }} "unset PYTHONWARNINGS && \
+                        ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+                        coverage combine && coverage xml"
     - name: Run regen
       timeout-minutes: 4
       run: |
@@ -257,7 +264,7 @@ jobs:
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
-      timeout-minutes: 20
+      timeout-minutes: 40
       run: |
         ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"

selfdrive/car/tests/test_models.py

@@ -24,6 +24,7 @@
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.route import SegmentName
+from openpilot.selfdrive.test.fuzzy_generation import FuzzyGenerator
 
 from panda.tests.libpanda import libpanda_py
 
@@ -179,6 +180,8 @@ def setUp(self):
     self.assertEqual(0, set_status, f"failed to set safetyModel {cfg}")
     self.safety.init_tests()
 
+    self.tx_fuzzy_ts_nanos = 0
+
   def test_car_params(self):
     if self.CP.dashcamOnly:
       self.skipTest("no need to check carParams for dashcamOnly")
@@ -368,6 +371,65 @@ def test_panda_safety_carstate_fuzzy(self, data):
         if self.safety.get_acc_main_on() != prev_panda_acc_main_on:
           self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on())
 
+  @pytest.mark.nocapture
+  @settings(max_examples=MAX_EXAMPLES, deadline=None,
+            phases=(Phase.reuse, Phase.generate, Phase.shrink))
+  @given(data=st.data())
+  def test_panda_safety_tx_fuzzy(self, data):
+    """
+    For each example, randomly imitate a carstate, send a randomly generated CarControl struct to
+    Openpilot and Panda. Check for matching logic between the two
+    """
+    if self.CP.dashcamOnly or self.CP.notCar:
+      self.skipTest("no need to check tx panda safety for dashcamOnly and notCar")
+
+    stop_frame = data.draw(st.integers(min_value=20, max_value=300))
+    for can in self.can_msgs[:stop_frame]:
+      self.CI.update(can_capnp_to_list((can.as_builder().to_bytes(), )))
+      for msg in filter(lambda m: m.src in range(64), can.can):
+        to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat)
+        self.safety.safety_rx_hook(to_send)
+
+    cc_msg = FuzzyGenerator.get_random_msg(data.draw, car.CarControl, real_floats=True)
+
+    # check for controls_allowed and set True if not already in these cases:
+    # - cancel button messages before controls_allowed
+    # - actuator commands before controls_allowed
+    # - resume button message before controls_allowed
+    controls_state = any([cc_msg["enabled"], cc_msg["latActive"], cc_msg["longActive"]])
+    panda_controls_allowed = any([controls_state, cc_msg["cruiseControl"]["resume"], cc_msg["cruiseControl"]["cancel"]])
+    self.safety.set_controls_allowed(panda_controls_allowed)
+    self.safety.set_cruise_engaged_prev(panda_controls_allowed) # ford: violation if cancel engage without cruise_engaged_prev
+
+    # relay_malfunction might be set during randomizing carState
+    if self.safety.get_relay_malfunction():
+      self.safety.set_relay_malfunction(False)
+
+    # no longitudinal controls if gas_pressed_prev
+    if self.safety.get_gas_pressed_prev():
+      cc_msg["longActive"] = False
+      cc_msg["enabled"] = False
+
+    if cc_msg["longActive"]:
+      cc_msg["enabled"] = True
+    else:
+      # follow logic of LoC in controlsd to set accel to 0 if longActive is False
+      cc_msg["actuators"]["accel"] = 0
+
+    CC = car.CarControl.new_message(**cc_msg).as_reader()
+
+    if self.CP.carName == "ford":
+      desired_angle_last = round(self.CI.CC.apply_curvature_last * 50000)
+      self.safety.set_angle_meas(desired_angle_last, desired_angle_last)
+
+    self.safety.set_timer(int(self.tx_fuzzy_ts_nanos / 1e3))
+    _, sendcans = self.CI.apply(CC, self.tx_fuzzy_ts_nanos)
+    self.tx_fuzzy_ts_nanos += DT_CTRL * 1e9
+
+    for addr, dat, bus in sendcans:
+      to_send = libpanda_py.make_CANPacket(addr, bus % 4, dat)
+      self.assertTrue(self.safety.safety_tx_hook(to_send), (addr, dat, bus))
+
   def test_panda_safety_carstate(self):
     """
       Assert that panda safety matches openpilot's carState