From 71dcc34805e64043aa5312326a66fff7c8ac4a58 Mon Sep 17 00:00:00 2001
From: Greg Hogan <gregjhogan@gmail.com>
Date: Sat, 25 Dec 2021 16:34:34 -0800
Subject: [PATCH 01/29] eliminate brands based on ECUs that respond to tester
 present

---
 selfdrive/car/ecu_addrs.py   | 64 ++++++++++++++++++++++++++++++++++++
 selfdrive/car/fw_versions.py | 27 +++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100755 selfdrive/car/ecu_addrs.py

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
new file mode 100755
index 00000000000000..1b6b25fcda56aa
--- /dev/null
+++ b/selfdrive/car/ecu_addrs.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+import traceback
+from typing import Any, Set
+
+import cereal.messaging as messaging
+from panda.python.uds import SERVICE_TYPE
+from selfdrive.boardd.boardd import can_list_to_can_capnp
+from selfdrive.swaglog import cloudlog
+
+# TODO: figure out type annotation for CanData
+def is_tester_present_response(msg: Any) -> bool:
+  # ISO-TP messages are always padded to 8 bytes
+  # tester present response is always a single frame
+  if len(msg.dat) == 8 and msg.dat[0] >= 1 and msg.dat[0] <= 7:
+    # success response
+    if msg.dat[1] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
+      return True
+    # error response
+    if msg.dat[1] == 0x7F and msg.dat[2] == SERVICE_TYPE.TESTER_PRESENT:
+      return True
+  return False
+
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float=0.1, debug: bool=False) -> Set[int]:
+  ecu_addrs = set()
+  try:
+    addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
+    msgs = [[addr, 0, bytes([SERVICE_TYPE.TESTER_PRESENT, 0x0]), bus] for addr in addr_list]
+    messaging.drain_sock(logcan)
+    sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
+    start_time = time.monotonic()
+    while time.monotonic() - start_time < timeout:
+      can_packets = messaging.drain_sock(logcan, wait_for_one=True)
+      for packet in can_packets:
+        for msg in packet.can:
+          if msg.src == bus and msg.address in addr_list and is_tester_present_response(msg):
+            if debug:
+              print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")
+              if msg.address in ecu_addrs:
+                print(f"Duplicate ECU address: {hex(msg.address)}")
+            ecu_addrs.add(msg.address)
+  except Exception:
+    cloudlog.warning(f"ECU addr scan exception: {traceback.format_exc()}")
+  return ecu_addrs
+
+if __name__ == "__main__":
+  import time
+  import argparse
+
+  parser = argparse.ArgumentParser(description='Get addresses of all ECUs')
+  parser.add_argument('--debug', action='store_true')
+  args = parser.parse_args()
+
+  logcan = messaging.sub_sock('can')
+  sendcan = messaging.pub_sock('sendcan')
+
+  time.sleep(1.0)
+
+  print("Getting ECU addresses ...")
+  ecu_addrs = get_ecu_addrs(logcan, sendcan, 1, debug=args.debug)
+
+  print()
+  print("Found ECUs on addresses:")
+  for addr in ecu_addrs:
+    print(f"  {hex(addr)}")
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 120174c4514a3e..c4fb99baa6f5f0 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -9,6 +9,7 @@
 
 import panda.python.uds as uds
 from cereal import car
+from selfdrive.car.ecu_addrs import get_ecu_addrs
 from selfdrive.car.fingerprints import FW_VERSIONS, get_attr_from_cars
 from selfdrive.car.isotp_parallel_query import IsoTpParallelQuery
 from selfdrive.car.toyota.values import CAR as TOYOTA
@@ -294,6 +295,29 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
   return exact_match, matches
 
+def get_brand_candidates(logcan, sendcan, bus, versions):
+  response_offset_by_brand = dict()
+  for brand, _, _, response_offset in REQUESTS:
+    response_offset_by_brand[brand] = response_offset
+
+  response_addrs_by_brand = dict()
+  for brand, brand_versions in versions.items():
+    if brand not in response_offset_by_brand:
+      continue
+
+    response_addrs_by_brand[brand] = set()
+    for c in brand_versions.values():
+      for _, addr, _ in c.keys():
+        response_addr = addr + response_offset_by_brand[brand]
+        response_addrs_by_brand[brand].add(response_addr)
+
+  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, bus)
+
+  brand_candidates = list()
+  for brand, brand_addrs in response_addrs_by_brand.items():
+    if brand_addrs.intersection(ecu_response_addrs) >= 4:
+      brand_candidates.append(brand)
+  return brand_candidates
 
 def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progress=False):
   ecu_types = {}
@@ -307,7 +331,10 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
   if extra is not None:
     versions.update(extra)
 
+  brand_candidates = get_brand_candidates(logcan, sendcan, bus, versions)
   for brand, brand_versions in versions.items():
+    if brand not in brand_candidates:
+      continue
     for c in brand_versions.values():
       for ecu_type, addr, sub_addr in c.keys():
         a = (brand, addr, sub_addr)

From 43bc6a62da07f58fc19944a577945ae360d6ea78 Mon Sep 17 00:00:00 2001
From: Greg Hogan <gregjhogan@gmail.com>
Date: Sat, 25 Dec 2021 18:07:38 -0800
Subject: [PATCH 02/29] make it work

---
 selfdrive/car/ecu_addrs.py   | 8 +++++---
 selfdrive/car/fw_versions.py | 4 ++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 1b6b25fcda56aa..38757050e34290 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import time
 import traceback
 from typing import Any, Set
 
@@ -20,11 +21,13 @@ def is_tester_present_response(msg: Any) -> bool:
       return True
   return False
 
-def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float=0.1, debug: bool=False) -> Set[int]:
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float=1, debug: bool=True) -> Set[int]:
   ecu_addrs = set()
   try:
     addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
-    msgs = [[addr, 0, bytes([SERVICE_TYPE.TESTER_PRESENT, 0x0]), bus] for addr in addr_list]
+    tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
+    msgs = [[addr, 0, tester_present, bus] for addr in addr_list]
+
     messaging.drain_sock(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
     start_time = time.monotonic()
@@ -43,7 +46,6 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus
   return ecu_addrs
 
 if __name__ == "__main__":
-  import time
   import argparse
 
   parser = argparse.ArgumentParser(description='Get addresses of all ECUs')
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index c4fb99baa6f5f0..f14d3f9a219a37 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -315,7 +315,7 @@ def get_brand_candidates(logcan, sendcan, bus, versions):
 
   brand_candidates = list()
   for brand, brand_addrs in response_addrs_by_brand.items():
-    if brand_addrs.intersection(ecu_response_addrs) >= 4:
+    if len(brand_addrs.intersection(ecu_response_addrs)) >= 4:
       brand_candidates.append(brand)
   return brand_candidates
 
@@ -333,7 +333,7 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
 
   brand_candidates = get_brand_candidates(logcan, sendcan, bus, versions)
   for brand, brand_versions in versions.items():
-    if brand not in brand_candidates:
+    if brand_candidates and brand not in brand_candidates:
       continue
     for c in brand_versions.values():
       for ecu_type, addr, sub_addr in c.keys():

From 5d0a4701a502b2bbd35b83cb96da7cfe84cdc141 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Tue, 31 May 2022 19:19:40 -0700
Subject: [PATCH 03/29] Add type hint for can message

Use make_can_msg
---
 selfdrive/car/ecu_addrs.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 38757050e34290..2598e248f3b2f1 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -1,18 +1,20 @@
 #!/usr/bin/env python3
+import capnp
 import time
 import traceback
-from typing import Any, Set
+from typing import Set
 
 import cereal.messaging as messaging
 from panda.python.uds import SERVICE_TYPE
+from selfdrive.car import make_can_msg
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from selfdrive.swaglog import cloudlog
 
-# TODO: figure out type annotation for CanData
-def is_tester_present_response(msg: Any) -> bool:
+
+def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader) -> bool:
   # ISO-TP messages are always padded to 8 bytes
   # tester present response is always a single frame
-  if len(msg.dat) == 8 and msg.dat[0] >= 1 and msg.dat[0] <= 7:
+  if len(msg.dat) == 8 and 1 <= msg.dat[0] <= 7:
     # success response
     if msg.dat[1] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
       return True
@@ -21,12 +23,13 @@ def is_tester_present_response(msg: Any) -> bool:
       return True
   return False
 
-def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float=1, debug: bool=True) -> Set[int]:
+
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[int]:
   ecu_addrs = set()
   try:
     addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
     tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
-    msgs = [[addr, 0, tester_present, bus] for addr in addr_list]
+    msgs = [make_can_msg(addr, tester_present, bus) for addr in addr_list]
 
     messaging.drain_sock(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
@@ -45,6 +48,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus
     cloudlog.warning(f"ECU addr scan exception: {traceback.format_exc()}")
   return ecu_addrs
 
+
 if __name__ == "__main__":
   import argparse
 

From ad40d4737d1d52bc6bc180782a3405cb8a207cad Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Tue, 31 May 2022 23:25:55 -0700
Subject: [PATCH 04/29] Only query for addresses in fingerprints, and account
 for different busses

---
 selfdrive/car/ecu_addrs.py   | 17 +++++++++++------
 selfdrive/car/fw_versions.py | 25 ++++++++++++++-----------
 2 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 2598e248f3b2f1..25d403b42e4a93 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -2,7 +2,7 @@
 import capnp
 import time
 import traceback
-from typing import Set
+from typing import Dict, Optional, Set
 
 import cereal.messaging as messaging
 from panda.python.uds import SERVICE_TYPE
@@ -24,12 +24,17 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader) -> boo
   return False
 
 
-def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[int]:
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, addr_bus_dict: Optional[Dict[int, int]] = None, bus: Optional[int] = None, timeout: float = 1, debug: bool = True) -> Set[int]:
+  assert not (addr_bus_dict is None and bus is None), "Need to specify either bus or address and bus dictionary"
+
   ecu_addrs = set()
   try:
-    addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
+    if addr_bus_dict is None:
+      addr_bus_dict = {0x700 + i: bus for i in range(256)}
+      addr_bus_dict.update({0x18da00f1 + (i << 8): bus for i in range(256)})
+
     tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
-    msgs = [make_can_msg(addr, tester_present, bus) for addr in addr_list]
+    msgs = [make_can_msg(addr, tester_present, bus) for addr, bus in addr_bus_dict.items()]
 
     messaging.drain_sock(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
@@ -38,7 +43,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus
       can_packets = messaging.drain_sock(logcan, wait_for_one=True)
       for packet in can_packets:
         for msg in packet.can:
-          if msg.src == bus and msg.address in addr_list and is_tester_present_response(msg):
+          if msg.address in addr_bus_dict and msg.src == addr_bus_dict[msg.address] and is_tester_present_response(msg):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")
               if msg.address in ecu_addrs:
@@ -62,7 +67,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus
   time.sleep(1.0)
 
   print("Getting ECU addresses ...")
-  ecu_addrs = get_ecu_addrs(logcan, sendcan, 1, debug=args.debug)
+  ecu_addrs = get_ecu_addrs(logcan, sendcan, bus=1, debug=args.debug)
 
   print()
   print("Found ECUs on addresses:")
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index f14d3f9a219a37..ce87f22d6ff626 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -295,27 +295,30 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
   return exact_match, matches
 
-def get_brand_candidates(logcan, sendcan, bus, versions):
-  response_offset_by_brand = dict()
-  for brand, _, _, response_offset in REQUESTS:
-    response_offset_by_brand[brand] = response_offset
+def get_brand_candidates(logcan, sendcan, versions):
+  response_offset_bus_by_brand = dict()
+  for r in REQUESTS:
+    # TODO: some brands have multiple response offsets
+    response_offset_bus_by_brand[r.brand] = (r.rx_offset, r.bus)
 
   response_addrs_by_brand = dict()
   for brand, brand_versions in versions.items():
-    if brand not in response_offset_by_brand:
+    if brand not in response_offset_bus_by_brand:
       continue
 
-    response_addrs_by_brand[brand] = set()
+    response_addrs_by_brand[brand] = {}
     for c in brand_versions.values():
       for _, addr, _ in c.keys():
-        response_addr = addr + response_offset_by_brand[brand]
-        response_addrs_by_brand[brand].add(response_addr)
+        response_addr = addr + response_offset_bus_by_brand[brand][0]
+        response_addrs_by_brand[brand][response_addr] = response_offset_bus_by_brand[brand][1]
 
-  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, bus)
+  # maps all addresses in versions to their request's bus
+  all_response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
+  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, all_response_addrs)
 
   brand_candidates = list()
   for brand, brand_addrs in response_addrs_by_brand.items():
-    if len(brand_addrs.intersection(ecu_response_addrs)) >= 4:
+    if len(set(brand_addrs).intersection(ecu_response_addrs)) >= 4:
       brand_candidates.append(brand)
   return brand_candidates
 
@@ -331,7 +334,7 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
   if extra is not None:
     versions.update(extra)
 
-  brand_candidates = get_brand_candidates(logcan, sendcan, bus, versions)
+  brand_candidates = get_brand_candidates(logcan, sendcan, versions)
   for brand, brand_versions in versions.items():
     if brand_candidates and brand not in brand_candidates:
       continue

From 55b837af91820a2607fa8223aa024bf5eeaf23a3 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Tue, 31 May 2022 23:47:09 -0700
Subject: [PATCH 05/29] These need to be addresses, not response addresses

---
 selfdrive/car/ecu_addrs.py   |  2 +-
 selfdrive/car/fw_versions.py | 15 ++++++++-------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 25d403b42e4a93..d38283c5013693 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -29,7 +29,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, add
 
   ecu_addrs = set()
   try:
-    if addr_bus_dict is None:
+    if bus is not None:
       addr_bus_dict = {0x700 + i: bus for i in range(256)}
       addr_bus_dict.update({0x18da00f1 + (i << 8): bus for i in range(256)})
 
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index ce87f22d6ff626..d5615bd43b04b4 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -301,24 +301,25 @@ def get_brand_candidates(logcan, sendcan, versions):
     # TODO: some brands have multiple response offsets
     response_offset_bus_by_brand[r.brand] = (r.rx_offset, r.bus)
 
-  response_addrs_by_brand = dict()
+  # maps all addresses in versions to their request's bus
+  addrs_by_brand = defaultdict(dict)
+  response_addrs_by_brand = defaultdict(set)
   for brand, brand_versions in versions.items():
     if brand not in response_offset_bus_by_brand:
       continue
 
-    response_addrs_by_brand[brand] = {}
     for c in brand_versions.values():
       for _, addr, _ in c.keys():
         response_addr = addr + response_offset_bus_by_brand[brand][0]
-        response_addrs_by_brand[brand][response_addr] = response_offset_bus_by_brand[brand][1]
+        response_addrs_by_brand[brand].add(response_addr)
+        addrs_by_brand[brand][addr] = response_offset_bus_by_brand[brand][0]
 
-  # maps all addresses in versions to their request's bus
-  all_response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
-  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, all_response_addrs)
+  addr_bus_dict = dict(addr for addr_bus in addrs_by_brand.values() for addr in addr_bus.items())
+  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, addr_bus_dict)
 
   brand_candidates = list()
   for brand, brand_addrs in response_addrs_by_brand.items():
-    if len(set(brand_addrs).intersection(ecu_response_addrs)) >= 4:
+    if len(brand_addrs.intersection(ecu_response_addrs)) >= 4:
       brand_candidates.append(brand)
   return brand_candidates
 

From 8e54436faec99fb9ab09610b4b562d975309fc92 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Wed, 1 Jun 2022 00:13:30 -0700
Subject: [PATCH 06/29] We need to listen to response addresses, not query
 addresses

---
 selfdrive/car/ecu_addrs.py   | 18 +++++++++---------
 selfdrive/car/fw_versions.py | 19 ++++++++++---------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index d38283c5013693..5e8c705a6812e1 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -24,17 +24,17 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader) -> boo
   return False
 
 
-def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, addr_bus_dict: Optional[Dict[int, int]] = None, bus: Optional[int] = None, timeout: float = 1, debug: bool = True) -> Set[int]:
-  assert not (addr_bus_dict is None and bus is None), "Need to specify either bus or address and bus dictionary"
+def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[int]:
+  addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
+  query_addrs = response_addrs = {addr: bus for addr in addr_list}
+  return get_ecu_addrs(logcan, sendcan, query_addrs, response_addrs, timeout=timeout, debug=debug)
 
+
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, query_addrs: Dict[int, int], response_addrs: Dict[int, int], timeout: float = 1, debug: bool = True) -> Set[int]:
   ecu_addrs = set()
   try:
-    if bus is not None:
-      addr_bus_dict = {0x700 + i: bus for i in range(256)}
-      addr_bus_dict.update({0x18da00f1 + (i << 8): bus for i in range(256)})
-
     tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
-    msgs = [make_can_msg(addr, tester_present, bus) for addr, bus in addr_bus_dict.items()]
+    msgs = [make_can_msg(addr, tester_present, bus) for addr, bus in query_addrs.items()]
 
     messaging.drain_sock(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
@@ -43,7 +43,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, add
       can_packets = messaging.drain_sock(logcan, wait_for_one=True)
       for packet in can_packets:
         for msg in packet.can:
-          if msg.address in addr_bus_dict and msg.src == addr_bus_dict[msg.address] and is_tester_present_response(msg):
+          if msg.address in response_addrs and msg.src == response_addrs[msg.address] and is_tester_present_response(msg):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")
               if msg.address in ecu_addrs:
@@ -67,7 +67,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, add
   time.sleep(1.0)
 
   print("Getting ECU addresses ...")
-  ecu_addrs = get_ecu_addrs(logcan, sendcan, bus=1, debug=args.debug)
+  ecu_addrs = get_all_ecu_addrs(logcan, sendcan, 1, debug=args.debug)
 
   print()
   print("Found ECUs on addresses:")
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index d5615bd43b04b4..59a915a63d8f1c 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -301,25 +301,26 @@ def get_brand_candidates(logcan, sendcan, versions):
     # TODO: some brands have multiple response offsets
     response_offset_bus_by_brand[r.brand] = (r.rx_offset, r.bus)
 
-  # maps all addresses in versions to their request's bus
-  addrs_by_brand = defaultdict(dict)
-  response_addrs_by_brand = defaultdict(set)
+  # maps query and response addresses to their request's bus
+  query_addrs_by_brand = defaultdict(dict)
+  response_addrs_by_brand = defaultdict(dict)
   for brand, brand_versions in versions.items():
     if brand not in response_offset_bus_by_brand:
       continue
 
     for c in brand_versions.values():
       for _, addr, _ in c.keys():
-        response_addr = addr + response_offset_bus_by_brand[brand][0]
-        response_addrs_by_brand[brand].add(response_addr)
-        addrs_by_brand[brand][addr] = response_offset_bus_by_brand[brand][0]
+        response_offset, bus = response_offset_bus_by_brand[brand]
+        response_addrs_by_brand[brand][addr + response_offset] = bus
+        query_addrs_by_brand[brand][addr] = bus
 
-  addr_bus_dict = dict(addr for addr_bus in addrs_by_brand.values() for addr in addr_bus.items())
-  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, addr_bus_dict)
+  query_addrs = dict(addr for addr_bus in query_addrs_by_brand.values() for addr in addr_bus.items())
+  response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
+  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, query_addrs, response_addrs)
 
   brand_candidates = list()
   for brand, brand_addrs in response_addrs_by_brand.items():
-    if len(brand_addrs.intersection(ecu_response_addrs)) >= 4:
+    if len(set(brand_addrs).intersection(ecu_response_addrs)) >= 4:
       brand_candidates.append(brand)
   return brand_candidates
 

From e981adde4e09c8330ba212d0c44fd0f8f60b0ca0 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Wed, 1 Jun 2022 00:16:07 -0700
Subject: [PATCH 07/29] add to files_common

---
 release/files_common | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release/files_common b/release/files_common
index 982194d774d87f..e537d3b742ee6f 100644
--- a/release/files_common
+++ b/release/files_common
@@ -101,6 +101,7 @@ selfdrive/car/interfaces.py
 selfdrive/car/vin.py
 selfdrive/car/disable_ecu.py
 selfdrive/car/fw_versions.py
+selfdrive/car/ecu_addrs.py
 selfdrive/car/isotp_parallel_query.py
 selfdrive/car/tests/__init__.py
 selfdrive/car/tests/test_car_interfaces.py

From 03ec08c83588c6d0517a0349a57afe78ed74e204 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Wed, 1 Jun 2022 11:46:18 -0700
Subject: [PATCH 08/29] Unused Optional Drain sock raw

---
 selfdrive/car/ecu_addrs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 5e8c705a6812e1..5c2f9fcd384a1f 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -2,7 +2,7 @@
 import capnp
 import time
 import traceback
-from typing import Dict, Optional, Set
+from typing import Dict, Set
 
 import cereal.messaging as messaging
 from panda.python.uds import SERVICE_TYPE
@@ -36,7 +36,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
     tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
     msgs = [make_can_msg(addr, tester_present, bus) for addr, bus in query_addrs.items()]
 
-    messaging.drain_sock(logcan)
+    messaging.drain_sock_raw(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
     start_time = time.monotonic()
     while time.monotonic() - start_time < timeout:

From 10d68a05492c410121878e4536af78ef52ed4a79 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Wed, 1 Jun 2022 11:54:57 -0700
Subject: [PATCH 09/29] add logging

---
 selfdrive/car/fw_versions.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 59a915a63d8f1c..43ecdb62e3cc3d 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -317,9 +317,11 @@ def get_brand_candidates(logcan, sendcan, versions):
   query_addrs = dict(addr for addr_bus in query_addrs_by_brand.values() for addr in addr_bus.items())
   response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
   ecu_response_addrs = get_ecu_addrs(logcan, sendcan, query_addrs, response_addrs)
+  cloudlog.event("ecu response addrs", ecu_response_addrs=ecu_response_addrs)
 
   brand_candidates = list()
   for brand, brand_addrs in response_addrs_by_brand.items():
+    cloudlog.event("candidate brand intersection", brand=brand, intersection=set(brand_addrs).intersection(ecu_response_addrs))
     if len(set(brand_addrs).intersection(ecu_response_addrs)) >= 4:
       brand_candidates.append(brand)
   return brand_candidates

From 250a89c8b43c38fe6df0ec5c09bf33240bee3eec Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Wed, 1 Jun 2022 17:18:15 -0700
Subject: [PATCH 10/29] only query essential ecus

comments
---
 selfdrive/car/ecu_addrs.py   |  5 +++--
 selfdrive/car/fw_versions.py | 12 ++++++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 5c2f9fcd384a1f..e34e3ae2ebc78f 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -10,6 +10,8 @@
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from selfdrive.swaglog import cloudlog
 
+TESTER_PRESENT_DAT = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
+
 
 def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader) -> bool:
   # ISO-TP messages are always padded to 8 bytes
@@ -33,8 +35,7 @@ def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket,
 def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, query_addrs: Dict[int, int], response_addrs: Dict[int, int], timeout: float = 1, debug: bool = True) -> Set[int]:
   ecu_addrs = set()
   try:
-    tester_present = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
-    msgs = [make_can_msg(addr, tester_present, bus) for addr, bus in query_addrs.items()]
+    msgs = [make_can_msg(addr, TESTER_PRESENT_DAT, bus) for addr, bus in query_addrs.items()]
 
     messaging.drain_sock_raw(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 43ecdb62e3cc3d..353145aaf3cb25 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -16,6 +16,7 @@
 from selfdrive.swaglog import cloudlog
 
 Ecu = car.CarParams.Ecu
+ESSENTIAL_ECUS = [Ecu.engine, Ecu.eps, Ecu.esp, Ecu.fwdRadar, Ecu.fwdCamera, Ecu.vsa]
 
 
 def p16(val):
@@ -258,7 +259,6 @@ def match_fw_to_car_exact(fw_versions_dict):
       ecu_type = ecu[0]
       addr = ecu[1:]
       found_version = fw_versions_dict.get(addr, None)
-      ESSENTIAL_ECUS = [Ecu.engine, Ecu.eps, Ecu.esp, Ecu.fwdRadar, Ecu.fwdCamera, Ecu.vsa]
       if ecu_type == Ecu.esp and candidate in (TOYOTA.RAV4, TOYOTA.COROLLA, TOYOTA.HIGHLANDER, TOYOTA.SIENNA, TOYOTA.LEXUS_IS) and found_version is None:
         continue
 
@@ -309,10 +309,14 @@ def get_brand_candidates(logcan, sendcan, versions):
       continue
 
     for c in brand_versions.values():
-      for _, addr, _ in c.keys():
+      for ecu, addr, _ in c.keys():
+        # reduce matches with ecus not critical to fingerprinting
+        if ecu not in ESSENTIAL_ECUS:
+          continue
+
         response_offset, bus = response_offset_bus_by_brand[brand]
-        response_addrs_by_brand[brand][addr + response_offset] = bus
         query_addrs_by_brand[brand][addr] = bus
+        response_addrs_by_brand[brand][addr + response_offset] = bus
 
   query_addrs = dict(addr for addr_bus in query_addrs_by_brand.values() for addr in addr_bus.items())
   response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
@@ -330,7 +334,7 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
   ecu_types = {}
 
   # Extract ECU addresses to query from fingerprints
-  # ECUs using a subadress need be queried one by one, the rest can be done in parallel
+  # ECUs using a subaddress need be queried one by one, the rest can be done in parallel
   addrs = []
   parallel_addrs = []
 

From 2ee55ae418b91c7f373267d05d43cf17c6faae6b Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 15:19:58 -0700
Subject: [PATCH 11/29] simplify get_brand_candidates(), keep track of multiple
 request variants per make and request each subaddress

---
 selfdrive/car/ecu_addrs.py   | 55 ++++++++++++++++++++++-----------
 selfdrive/car/fw_versions.py | 60 +++++++++++++++++-------------------
 2 files changed, 66 insertions(+), 49 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index e34e3ae2ebc78f..f82cc1c344a02d 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -2,7 +2,7 @@
 import capnp
 import time
 import traceback
-from typing import Dict, Set
+from typing import Optional, Set, Tuple
 
 import cereal.messaging as messaging
 from panda.python.uds import SERVICE_TYPE
@@ -10,32 +10,47 @@
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from selfdrive.swaglog import cloudlog
 
-TESTER_PRESENT_DAT = bytes([0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0])
 
+def make_tester_present_msg(addr, bus, subaddr=None):
+  dat = [0x02, SERVICE_TYPE.TESTER_PRESENT, 0x0]
+  if subaddr is not None:
+    dat.insert(0, subaddr)
 
-def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader) -> bool:
+  dat += [0x0] * (8 - len(dat))
+  return make_can_msg(addr, dat, bus)
+
+
+def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subaddr: Optional[int] = None) -> bool:
   # ISO-TP messages are always padded to 8 bytes
   # tester present response is always a single frame
-  if len(msg.dat) == 8 and 1 <= msg.dat[0] <= 7:
+  dat_offset = 1 if subaddr is not None else 0
+  if len(msg.dat) == 8 and 1 <= msg.dat[dat_offset] <= 7:
     # success response
-    if msg.dat[1] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
+    if msg.dat[1 + dat_offset] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
       return True
     # error response
-    if msg.dat[1] == 0x7F and msg.dat[2] == SERVICE_TYPE.TESTER_PRESENT:
+    if msg.dat[1 + dat_offset] == 0x7F and msg.dat[2 + dat_offset] == SERVICE_TYPE.TESTER_PRESENT:
       return True
   return False
 
 
-def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[int]:
+def get_msg_subaddr(msg: capnp.lib.capnp._DynamicStructReader) -> Optional[int]:
+  if 1 <= msg.dat[1] <= 7:
+    return msg.dat[0]
+  return None
+
+
+def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[Tuple[int, Optional[int], int]]:
   addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
-  query_addrs = response_addrs = {addr: bus for addr in addr_list}
-  return get_ecu_addrs(logcan, sendcan, query_addrs, response_addrs, timeout=timeout, debug=debug)
+  queries = responses = {(addr, None, bus) for addr in addr_list}
+  return get_ecu_addrs(logcan, sendcan, queries, responses, timeout=timeout, debug=debug)
 
 
-def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, query_addrs: Dict[int, int], response_addrs: Dict[int, int], timeout: float = 1, debug: bool = True) -> Set[int]:
-  ecu_addrs = set()
+def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, queries: Set[Tuple[int, Optional[int], int]],
+                  responses: Set[Tuple[int, Optional[int], int]], timeout: float = 1, debug: bool = True) -> Set[Tuple[int, Optional[int], int]]:
+  ecu_responses: Set[Tuple[int, Optional[int], int]] = set()  # set((addr, subaddr, bus),)
   try:
-    msgs = [make_can_msg(addr, TESTER_PRESENT_DAT, bus) for addr, bus in query_addrs.items()]
+    msgs = [make_tester_present_msg(addr, bus, subaddr) for addr, subaddr, bus in queries]
 
     messaging.drain_sock_raw(logcan)
     sendcan.send(can_list_to_can_capnp(msgs, msgtype='sendcan'))
@@ -44,15 +59,16 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
       can_packets = messaging.drain_sock(logcan, wait_for_one=True)
       for packet in can_packets:
         for msg in packet.can:
-          if msg.address in response_addrs and msg.src == response_addrs[msg.address] and is_tester_present_response(msg):
+          subaddr = get_msg_subaddr(msg)
+          if (msg.address, subaddr, msg.bus) in responses and is_tester_present_response(msg, subaddr):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")
-              if msg.address in ecu_addrs:
+              if (msg.address, subaddr, msg.bus) in ecu_responses:
                 print(f"Duplicate ECU address: {hex(msg.address)}")
-            ecu_addrs.add(msg.address)
+            ecu_responses.add((msg.address, subaddr, msg.bus))
   except Exception:
     cloudlog.warning(f"ECU addr scan exception: {traceback.format_exc()}")
-  return ecu_addrs
+  return ecu_responses
 
 
 if __name__ == "__main__":
@@ -72,5 +88,8 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
 
   print()
   print("Found ECUs on addresses:")
-  for addr in ecu_addrs:
-    print(f"  {hex(addr)}")
+  for addr, subaddr, bus in ecu_addrs:
+    msg = f"  0x{hex(addr)}"
+    if subaddr is not None:
+      msg += f" (sub-address: 0x{hex(subaddr)})"
+    print(msg)
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 9cb1657876733c..a4a5f604c7cb74 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -3,7 +3,7 @@
 import traceback
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, List
+from typing import Any, Dict, List, Optional, Set, Tuple
 from tqdm import tqdm
 
 import panda.python.uds as uds
@@ -297,40 +297,38 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
   return exact_match, matches
 
+
 def get_brand_candidates(logcan, sendcan, versions):
-  response_offset_bus_by_brand = dict()
+  queries: Set[Tuple[int, Optional[int], int]] = set()  # set((addr, subaddr, bus),)
+  response_to_brand: Dict[Tuple[int, Optional[int], int], str] = dict()  # {(response_addr, subaddr, bus): brand,}
   for r in REQUESTS:
-    # TODO: some brands have multiple response offsets
-    response_offset_bus_by_brand[r.brand] = (r.rx_offset, r.bus)
-
-  # maps query and response addresses to their request's bus
-  query_addrs_by_brand = defaultdict(dict)
-  response_addrs_by_brand = defaultdict(dict)
-  for brand, brand_versions in versions.items():
-    if brand not in response_offset_bus_by_brand:
+    if r.brand not in versions:
       continue
 
-    for c in brand_versions.values():
-      for ecu, addr, _ in c.keys():
-        # reduce matches with ecus not critical to fingerprinting
-        if ecu not in ESSENTIAL_ECUS:
-          continue
-
-        response_offset, bus = response_offset_bus_by_brand[brand]
-        query_addrs_by_brand[brand][addr] = bus
-        response_addrs_by_brand[brand][addr + response_offset] = bus
-
-  query_addrs = dict(addr for addr_bus in query_addrs_by_brand.values() for addr in addr_bus.items())
-  response_addrs = dict(addr for addr_bus in response_addrs_by_brand.values() for addr in addr_bus.items())
-  ecu_response_addrs = get_ecu_addrs(logcan, sendcan, query_addrs, response_addrs)
-  cloudlog.event("ecu response addrs", ecu_response_addrs=ecu_response_addrs)
-
-  brand_candidates = list()
-  for brand, brand_addrs in response_addrs_by_brand.items():
-    cloudlog.event("candidate brand intersection", brand=brand, intersection=set(brand_addrs).intersection(ecu_response_addrs))
-    if len(set(brand_addrs).intersection(ecu_response_addrs)) >= 4:
-      brand_candidates.append(brand)
-  return brand_candidates
+    for brand_versions in versions[r.brand].values():
+      for ecu_type, addr, subaddr in brand_versions:
+        # Only query ecus in whitelist if whitelist is not empty
+        if len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus:
+          # Build set of queries
+          queries.add((addr, subaddr, r.bus))
+
+          # Store map from response to brand
+          response_addr = addr + r.rx_offset
+          response_to_brand[(response_addr, subaddr, r.bus)] = r.brand
+
+  ecu_responses = get_ecu_addrs(logcan, sendcan, queries, set(response_to_brand.keys()))
+  cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
+
+  brand_candidates = defaultdict(set)
+  for response in ecu_responses:
+    if response in response_to_brand:
+      brand_candidates[response_to_brand[response]].add(response)  # address, subaddr, bus
+
+  for brand, responses in brand_candidates.items():
+    cloudlog.event(f"{brand} responses: {responses}")
+
+  return [brand for brand, responses in brand_candidates.items() if len(responses) >= 3]
+
 
 def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progress=False):
   ecu_types = {}

From 5793a1e032a9f67e5a9734d1ec43e29af45a9584 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 15:23:58 -0700
Subject: [PATCH 12/29] fixes

make dat bytes

bus is src

Fix check
---
 selfdrive/car/ecu_addrs.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index f82cc1c344a02d..a53572e170412f 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -17,7 +17,7 @@ def make_tester_present_msg(addr, bus, subaddr=None):
     dat.insert(0, subaddr)
 
   dat += [0x0] * (8 - len(dat))
-  return make_can_msg(addr, dat, bus)
+  return make_can_msg(addr, bytes(dat), bus)
 
 
 def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subaddr: Optional[int] = None) -> bool:
@@ -35,7 +35,7 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subadd
 
 
 def get_msg_subaddr(msg: capnp.lib.capnp._DynamicStructReader) -> Optional[int]:
-  if 1 <= msg.dat[1] <= 7:
+  if len(msg.dat) == 8 and 1 <= msg.dat[1] <= 7:
     return msg.dat[0]
   return None
 
@@ -60,12 +60,12 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
       for packet in can_packets:
         for msg in packet.can:
           subaddr = get_msg_subaddr(msg)
-          if (msg.address, subaddr, msg.bus) in responses and is_tester_present_response(msg, subaddr):
+          if (msg.address, subaddr, msg.src) in responses and is_tester_present_response(msg, subaddr):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")
-              if (msg.address, subaddr, msg.bus) in ecu_responses:
+              if (msg.address, subaddr, msg.src) in ecu_responses:
                 print(f"Duplicate ECU address: {hex(msg.address)}")
-            ecu_responses.add((msg.address, subaddr, msg.bus))
+            ecu_responses.add((msg.address, subaddr, msg.src))
   except Exception:
     cloudlog.warning(f"ECU addr scan exception: {traceback.format_exc()}")
   return ecu_responses

From c4ffd70ff25c15c9a53388a8ea7dc2d52c6d8af8 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 16:29:01 -0700
Subject: [PATCH 13/29] (addr, subaddr, bus) can be common across brands, add a
 match to each brand

---
 selfdrive/car/fw_versions.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index a4a5f604c7cb74..926ac9b9d2fa44 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -300,7 +300,7 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
 def get_brand_candidates(logcan, sendcan, versions):
   queries: Set[Tuple[int, Optional[int], int]] = set()  # set((addr, subaddr, bus),)
-  response_to_brand: Dict[Tuple[int, Optional[int], int], str] = dict()  # {(response_addr, subaddr, bus): brand,}
+  response_to_brand: Dict[Tuple[int, Optional[int], int], Set[str]] = defaultdict(set)  # {(response_addr, subaddr, bus): set(brand,),}
   for r in REQUESTS:
     if r.brand not in versions:
       continue
@@ -314,15 +314,17 @@ def get_brand_candidates(logcan, sendcan, versions):
 
           # Store map from response to brand
           response_addr = addr + r.rx_offset
-          response_to_brand[(response_addr, subaddr, r.bus)] = r.brand
+          response_to_brand[(response_addr, subaddr, r.bus)].add(r.brand)
 
+  print(response_to_brand)
   ecu_responses = get_ecu_addrs(logcan, sendcan, queries, set(response_to_brand.keys()))
   cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
 
   brand_candidates = defaultdict(set)
   for response in ecu_responses:
     if response in response_to_brand:
-      brand_candidates[response_to_brand[response]].add(response)  # address, subaddr, bus
+      for brand in response_to_brand[response]:
+        brand_candidates[brand].add(response)  # address, subaddr, bus
 
   for brand, responses in brand_candidates.items():
     cloudlog.event(f"{brand} responses: {responses}")

From c445b98b88381a338505d0d63ac42cc5576bbab0 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 16:44:41 -0700
Subject: [PATCH 14/29] fix length

---
 selfdrive/car/ecu_addrs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index a53572e170412f..8a99301da68b65 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -35,7 +35,7 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subadd
 
 
 def get_msg_subaddr(msg: capnp.lib.capnp._DynamicStructReader) -> Optional[int]:
-  if len(msg.dat) == 8 and 1 <= msg.dat[1] <= 7:
+  if len(msg.dat) == 8 and 1 <= msg.dat[1] <= 6:
     return msg.dat[0]
   return None
 

From 48e96517e5acf7f1edc8a7a7662aaff54b2553fb Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 18:06:29 -0700
Subject: [PATCH 15/29] query subaddrs in sequence

---
 selfdrive/car/fw_versions.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 926ac9b9d2fa44..196eaf24f18258 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -299,7 +299,8 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
 
 def get_brand_candidates(logcan, sendcan, versions):
-  queries: Set[Tuple[int, Optional[int], int]] = set()  # set((addr, subaddr, bus),)
+  queries = list()  # set((addr, subaddr, bus),)
+  parallel_queries = list()  # set((addr, subaddr, bus),)
   response_to_brand: Dict[Tuple[int, Optional[int], int], Set[str]] = defaultdict(set)  # {(response_addr, subaddr, bus): set(brand,),}
   for r in REQUESTS:
     if r.brand not in versions:
@@ -309,16 +310,27 @@ def get_brand_candidates(logcan, sendcan, versions):
       for ecu_type, addr, subaddr in brand_versions:
         # Only query ecus in whitelist if whitelist is not empty
         if len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus:
+          a = (addr, subaddr, r.bus)
           # Build set of queries
-          queries.add((addr, subaddr, r.bus))
+          if subaddr is None:
+            if a not in parallel_queries:
+              parallel_queries.append(a)
+          else:
+            if [a] not in queries:
+              queries.append([a])
 
           # Store map from response to brand
           response_addr = addr + r.rx_offset
           response_to_brand[(response_addr, subaddr, r.bus)].add(r.brand)
 
-  print(response_to_brand)
-  ecu_responses = get_ecu_addrs(logcan, sendcan, queries, set(response_to_brand.keys()))
+  queries.insert(0, parallel_queries)
+
+  ecu_responses: Set[Tuple[int, Optional[int], int]] = set()
+  for query in queries:
+    ecu_responses.update(get_ecu_addrs(logcan, sendcan, {query}, set(response_to_brand.keys())))
+
   cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
+  print("ecu responses", ecu_responses)
 
   brand_candidates = defaultdict(set)
   for response in ecu_responses:
@@ -328,6 +340,7 @@ def get_brand_candidates(logcan, sendcan, versions):
 
   for brand, responses in brand_candidates.items():
     cloudlog.event(f"{brand} responses: {responses}")
+    print(f"{brand} responses: {responses}")
 
   return [brand for brand, responses in brand_candidates.items() if len(responses) >= 3]
 

From a24077450aada8625126cfa79f75fdf940ebb7f4 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 19:31:24 -0700
Subject: [PATCH 16/29] fix

---
 selfdrive/car/fw_versions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 196eaf24f18258..a0a73a210d94c7 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -327,7 +327,7 @@ def get_brand_candidates(logcan, sendcan, versions):
 
   ecu_responses: Set[Tuple[int, Optional[int], int]] = set()
   for query in queries:
-    ecu_responses.update(get_ecu_addrs(logcan, sendcan, {query}, set(response_to_brand.keys())))
+    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), set(response_to_brand.keys())))
 
   cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
   print("ecu responses", ecu_responses)

From 9cad5c2b3c43db13f5ddc00ec811f9d14d46ff82 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 6 Jun 2022 23:16:41 -0700
Subject: [PATCH 17/29] candidate if a platform is a subset of responding ecu
 addresses

comment

comment
---
 selfdrive/car/fw_versions.py | 43 ++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index a0a73a210d94c7..527fb18e502bdc 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -299,50 +299,51 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
 
 
 def get_brand_candidates(logcan, sendcan, versions):
-  queries = list()  # set((addr, subaddr, bus),)
-  parallel_queries = list()  # set((addr, subaddr, bus),)
-  response_to_brand: Dict[Tuple[int, Optional[int], int], Set[str]] = defaultdict(set)  # {(response_addr, subaddr, bus): set(brand,),}
+  queries = list()
+  parallel_queries = list()
+  responses = set()
   for r in REQUESTS:
     if r.brand not in versions:
       continue
 
     for brand_versions in versions[r.brand].values():
-      for ecu_type, addr, subaddr in brand_versions:
+      for ecu_type, addr, sub_addr in brand_versions:
         # Only query ecus in whitelist if whitelist is not empty
-        if len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus:
-          a = (addr, subaddr, r.bus)
+        if (len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus) and ecu_type in ESSENTIAL_ECUS:
+          a = (addr, sub_addr, r.bus)
           # Build set of queries
-          if subaddr is None:
+          if sub_addr is None:
             if a not in parallel_queries:
               parallel_queries.append(a)
           else:
             if [a] not in queries:
               queries.append([a])
 
-          # Store map from response to brand
+          # Build set of expected responses to filter
           response_addr = addr + r.rx_offset
-          response_to_brand[(response_addr, subaddr, r.bus)].add(r.brand)
+          responses.add((response_addr, sub_addr, r.bus))
 
   queries.insert(0, parallel_queries)
 
   ecu_responses: Set[Tuple[int, Optional[int], int]] = set()
   for query in queries:
-    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), set(response_to_brand.keys())))
+    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses))
 
   cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
   print("ecu responses", ecu_responses)
 
-  brand_candidates = defaultdict(set)
-  for response in ecu_responses:
-    if response in response_to_brand:
-      for brand in response_to_brand[response]:
-        brand_candidates[brand].add(response)  # address, subaddr, bus
-
-  for brand, responses in brand_candidates.items():
-    cloudlog.event(f"{brand} responses: {responses}")
-    print(f"{brand} responses: {responses}")
-
-  return [brand for brand, responses in brand_candidates.items() if len(responses) >= 3]
+  brand_candidates = set()
+  for r in REQUESTS:
+    ecu_addrs = {(addr - r.rx_offset, sub_addr) for addr, sub_addr, _ in ecu_responses}
+    for candidate, candidate_fw in versions[r.brand].items():
+      # brand is a candidate if any of its platforms is a complete subset of the response ecus
+      candidate_addrs = {(addr, sub_addr) for ecu_type, addr, sub_addr in candidate_fw.keys() if ecu_type in ESSENTIAL_ECUS}
+      if len(ecu_addrs.intersection(candidate_addrs)) == len(candidate_addrs):
+        # print('Candidate: {}'.format(candidate))
+        brand_candidates.add(r.brand)
+
+  print('Brand candidates:', brand_candidates)
+  return brand_candidates
 
 
 def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progress=False):

From 5a77da39b165636741d364c72ea1895623533982 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Thu, 16 Jun 2022 10:29:22 -0700
Subject: [PATCH 18/29] do logging for shadow mode

---
 selfdrive/car/fw_versions.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 3d4f37569f2d3b..c3832b3891afef 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -329,8 +329,7 @@ def get_brand_candidates(logcan, sendcan, versions):
   for query in queries:
     ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses))
 
-  cloudlog.event("ecu responses", ecu_response_addrs=ecu_responses)
-  print("ecu responses", ecu_responses)
+  cloudlog.event("ecu responses", ecu_responses=ecu_responses)
 
   brand_candidates = set()
   for r in REQUESTS:
@@ -339,10 +338,8 @@ def get_brand_candidates(logcan, sendcan, versions):
       # brand is a candidate if any of its platforms is a complete subset of the response ecus
       candidate_addrs = {(addr, sub_addr) for ecu_type, addr, sub_addr in candidate_fw.keys() if ecu_type in ESSENTIAL_ECUS}
       if len(ecu_addrs.intersection(candidate_addrs)) == len(candidate_addrs):
-        # print('Candidate: {}'.format(candidate))
         brand_candidates.add(r.brand)
 
-  print('Brand candidates:', brand_candidates)
   return brand_candidates
 
 
@@ -358,10 +355,11 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
   if extra is not None:
     versions.update(extra)
 
+  # log brand candidates
   brand_candidates = get_brand_candidates(logcan, sendcan, versions)
+  cloudlog.event("brand candidates", ecu_response_addrs=brand_candidates)
+
   for brand, brand_versions in versions.items():
-    if brand_candidates and brand not in brand_candidates:
-      continue
     for c in brand_versions.values():
       for ecu_type, addr, sub_addr in c.keys():
         a = (brand, addr, sub_addr)

From d48a8855cbfd36292f78eeca1d29c19bc97d09ad Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 14:23:28 -0700
Subject: [PATCH 19/29] log responses so we can calculate candidates offline

---
 selfdrive/car/car_helpers.py |  8 +++++---
 selfdrive/car/ecu_addrs.py   |  2 +-
 selfdrive/car/fw_versions.py | 29 ++++++-----------------------
 3 files changed, 12 insertions(+), 27 deletions(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 3bfd99f88def6f..976d01d9e5a6cd 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -8,7 +8,7 @@
 from selfdrive.car.interfaces import get_interface_attr
 from selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars
 from selfdrive.car.vin import get_vin, VIN_UNKNOWN
-from selfdrive.car.fw_versions import get_fw_versions, match_fw_to_car
+from selfdrive.car.fw_versions import get_fw_versions, match_fw_to_car, get_present_ecus
 from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint
@@ -93,16 +93,18 @@ def fingerprint(logcan, sendcan):
     if cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
       cloudlog.warning("Using cached CarParams")
       vin = cached_params.carVin
+      ecu_responses = set()
       car_fw = list(cached_params.carFw)
     else:
       cloudlog.warning("Getting VIN & FW versions")
       _, vin = get_vin(logcan, sendcan, bus)
+      ecu_responses = get_present_ecus(logcan, sendcan, get_interface_attr('FW_VERSIONS', ignore_none=True))
       car_fw = get_fw_versions(logcan, sendcan)
 
     exact_fw_match, fw_candidates = match_fw_to_car(car_fw)
   else:
     vin = VIN_UNKNOWN
-    exact_fw_match, fw_candidates, car_fw = True, set(), []
+    exact_fw_match, fw_candidates, car_fw, ecu_responses = True, set(), [], set()
 
   if len(vin) != 17:
     cloudlog.event("Malformed VIN", vin=vin, error=True)
@@ -164,7 +166,7 @@ def fingerprint(logcan, sendcan):
     source = car.CarParams.FingerprintSource.fixed
 
   cloudlog.event("fingerprinted", car_fingerprint=car_fingerprint,
-                 source=source, fuzzy=not exact_match, fw_count=len(car_fw))
+                 source=source, fuzzy=not exact_match, fw_count=len(car_fw), ecu_responses=ecu_responses)
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
 
diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 8a99301da68b65..ee268cdf1993c5 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -8,7 +8,7 @@
 from panda.python.uds import SERVICE_TYPE
 from selfdrive.car import make_can_msg
 from selfdrive.boardd.boardd import can_list_to_can_capnp
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 def make_tester_present_msg(addr, bus, subaddr=None):
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 1a0e5db54e4d34..b6d3397e14de3a 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -300,7 +300,7 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
   return exact_match, matches
 
 
-def get_brand_candidates(logcan, sendcan, versions):
+def get_present_ecus(logcan, sendcan, versions):
   queries = list()
   parallel_queries = list()
   responses = set()
@@ -311,38 +311,25 @@ def get_brand_candidates(logcan, sendcan, versions):
     for brand_versions in versions[r.brand].values():
       for ecu_type, addr, sub_addr in brand_versions:
         # Only query ecus in whitelist if whitelist is not empty
-        if (len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus) and ecu_type in ESSENTIAL_ECUS:
+        if len(r.whitelist_ecus) == 0 or ecu_type in r.whitelist_ecus:
           a = (addr, sub_addr, r.bus)
           # Build set of queries
           if sub_addr is None:
             if a not in parallel_queries:
               parallel_queries.append(a)
-          else:
+          else:  # subaddresses must be queried one by one
             if [a] not in queries:
               queries.append([a])
 
           # Build set of expected responses to filter
-          response_addr = addr + r.rx_offset
-          responses.add((response_addr, sub_addr, r.bus))
+          responses.add((addr + r.rx_offset, sub_addr, r.bus))
 
   queries.insert(0, parallel_queries)
 
   ecu_responses: Set[Tuple[int, Optional[int], int]] = set()
   for query in queries:
-    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses))
-
-  cloudlog.event("ecu responses", ecu_responses=ecu_responses)
-
-  brand_candidates = set()
-  for r in REQUESTS:
-    ecu_addrs = {(addr - r.rx_offset, sub_addr) for addr, sub_addr, _ in ecu_responses}
-    for candidate, candidate_fw in versions[r.brand].items():
-      # brand is a candidate if any of its platforms is a complete subset of the response ecus
-      candidate_addrs = {(addr, sub_addr) for ecu_type, addr, sub_addr in candidate_fw.keys() if ecu_type in ESSENTIAL_ECUS}
-      if len(ecu_addrs.intersection(candidate_addrs)) == len(candidate_addrs):
-        brand_candidates.add(r.brand)
-
-  return brand_candidates
+    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses, timeout=0.2))
+  return ecu_responses
 
 
 def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progress=False):
@@ -357,10 +344,6 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
   if extra is not None:
     versions.update(extra)
 
-  # log brand candidates
-  brand_candidates = get_brand_candidates(logcan, sendcan, versions)
-  cloudlog.event("brand candidates", ecu_response_addrs=brand_candidates)
-
   for brand, brand_versions in versions.items():
     for c in brand_versions.values():
       for ecu_type, addr, sub_addr in c.keys():

From 490912c3cd9097b10aea2449d2a29d6a61a3a3ee Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 14:51:15 -0700
Subject: [PATCH 20/29] get has_subaddress from response set

---
 selfdrive/car/ecu_addrs.py | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index ee268cdf1993c5..93dab8184b1e6a 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -16,7 +16,7 @@ def make_tester_present_msg(addr, bus, subaddr=None):
   if subaddr is not None:
     dat.insert(0, subaddr)
 
-  dat += [0x0] * (8 - len(dat))
+  dat.extend([0x0] * (8 - len(dat)))
   return make_can_msg(addr, bytes(dat), bus)
 
 
@@ -26,20 +26,14 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subadd
   dat_offset = 1 if subaddr is not None else 0
   if len(msg.dat) == 8 and 1 <= msg.dat[dat_offset] <= 7:
     # success response
-    if msg.dat[1 + dat_offset] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
+    if msg.dat[dat_offset + 1] == (SERVICE_TYPE.TESTER_PRESENT + 0x40):
       return True
     # error response
-    if msg.dat[1 + dat_offset] == 0x7F and msg.dat[2 + dat_offset] == SERVICE_TYPE.TESTER_PRESENT:
+    if msg.dat[dat_offset + 1] == 0x7F and msg.dat[dat_offset + 2] == SERVICE_TYPE.TESTER_PRESENT:
       return True
   return False
 
 
-def get_msg_subaddr(msg: capnp.lib.capnp._DynamicStructReader) -> Optional[int]:
-  if len(msg.dat) == 8 and 1 <= msg.dat[1] <= 6:
-    return msg.dat[0]
-  return None
-
-
 def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[Tuple[int, Optional[int], int]]:
   addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
   queries = responses = {(addr, None, bus) for addr in addr_list}
@@ -59,7 +53,8 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
       can_packets = messaging.drain_sock(logcan, wait_for_one=True)
       for packet in can_packets:
         for msg in packet.can:
-          subaddr = get_msg_subaddr(msg)
+          has_subaddr = any([r[0] == msg.address and r[1] is not None and r[2] == msg.src for r in responses])
+          subaddr = msg.dat[0] if has_subaddr else None
           if (msg.address, subaddr, msg.src) in responses and is_tester_present_response(msg, subaddr):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")

From bb1d614b56481afc59df813b68e2e501e31b3e7d Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 15:05:33 -0700
Subject: [PATCH 21/29] one liner

---
 selfdrive/car/ecu_addrs.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 93dab8184b1e6a..494724f7cca13f 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -53,8 +53,7 @@ def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, que
       can_packets = messaging.drain_sock(logcan, wait_for_one=True)
       for packet in can_packets:
         for msg in packet.can:
-          has_subaddr = any([r[0] == msg.address and r[1] is not None and r[2] == msg.src for r in responses])
-          subaddr = msg.dat[0] if has_subaddr else None
+          subaddr = None if (msg.address, None, msg.src) in responses else msg.dat[0]
           if (msg.address, subaddr, msg.src) in responses and is_tester_present_response(msg, subaddr):
             if debug:
               print(f"CAN-RX: {hex(msg.address)} - 0x{bytes.hex(msg.dat)}")

From ea5a3ac44eaf365f482f27b2f9930e54153e0c31 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 15:20:17 -0700
Subject: [PATCH 22/29] fix mypy

---
 selfdrive/car/ecu_addrs.py   | 5 +++--
 selfdrive/car/fw_versions.py | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/selfdrive/car/ecu_addrs.py b/selfdrive/car/ecu_addrs.py
index 494724f7cca13f..267701509a6926 100755
--- a/selfdrive/car/ecu_addrs.py
+++ b/selfdrive/car/ecu_addrs.py
@@ -36,12 +36,13 @@ def is_tester_present_response(msg: capnp.lib.capnp._DynamicStructReader, subadd
 
 def get_all_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, bus: int, timeout: float = 1, debug: bool = True) -> Set[Tuple[int, Optional[int], int]]:
   addr_list = [0x700 + i for i in range(256)] + [0x18da00f1 + (i << 8) for i in range(256)]
-  queries = responses = {(addr, None, bus) for addr in addr_list}
+  queries: Set[Tuple[int, Optional[int], int]] = {(addr, None, bus) for addr in addr_list}
+  responses = queries
   return get_ecu_addrs(logcan, sendcan, queries, responses, timeout=timeout, debug=debug)
 
 
 def get_ecu_addrs(logcan: messaging.SubSocket, sendcan: messaging.PubSocket, queries: Set[Tuple[int, Optional[int], int]],
-                  responses: Set[Tuple[int, Optional[int], int]], timeout: float = 1, debug: bool = True) -> Set[Tuple[int, Optional[int], int]]:
+                  responses: Set[Tuple[int, Optional[int], int]], timeout: float = 1, debug: bool = False) -> Set[Tuple[int, Optional[int], int]]:
   ecu_responses: Set[Tuple[int, Optional[int], int]] = set()  # set((addr, subaddr, bus),)
   try:
     msgs = [make_tester_present_msg(addr, bus, subaddr) for addr, subaddr, bus in queries]
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index b6d3397e14de3a..4970b4d801e425 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -3,7 +3,7 @@
 import traceback
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, List, Optional, Set, Tuple
 from tqdm import tqdm
 
 import panda.python.uds as uds

From b7b633c4986109c51abc29708a994fd4dc8d731d Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 15:29:45 -0700
Subject: [PATCH 23/29] set to default at top

---
 selfdrive/car/car_helpers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 976d01d9e5a6cd..175c1061714d64 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -79,6 +79,7 @@ def _get_interface_names() -> Dict[str, List[str]]:
 def fingerprint(logcan, sendcan):
   fixed_fingerprint = os.environ.get('FINGERPRINT', "")
   skip_fw_query = os.environ.get('SKIP_FW_QUERY', False)
+  ecu_responses = set()
 
   if not fixed_fingerprint and not skip_fw_query:
     # Vin query only reliably works thorugh OBDII
@@ -93,7 +94,6 @@ def fingerprint(logcan, sendcan):
     if cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
       cloudlog.warning("Using cached CarParams")
       vin = cached_params.carVin
-      ecu_responses = set()
       car_fw = list(cached_params.carFw)
     else:
       cloudlog.warning("Getting VIN & FW versions")
@@ -104,7 +104,7 @@ def fingerprint(logcan, sendcan):
     exact_fw_match, fw_candidates = match_fw_to_car(car_fw)
   else:
     vin = VIN_UNKNOWN
-    exact_fw_match, fw_candidates, car_fw, ecu_responses = True, set(), [], set()
+    exact_fw_match, fw_candidates, car_fw = True, set(), []
 
   if len(vin) != 17:
     cloudlog.event("Malformed VIN", vin=vin, error=True)

From 6211c5ea99cd6f4622ae718a680bc75ed9e3f452 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 15:30:42 -0700
Subject: [PATCH 24/29] always log for now

---
 selfdrive/car/car_helpers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 175c1061714d64..10f9348b520762 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -91,7 +91,7 @@ def fingerprint(logcan, sendcan):
       if cached_params.carName == "mock":
         cached_params = None
 
-    if cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
+    if False:  # cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
       cloudlog.warning("Using cached CarParams")
       vin = cached_params.carVin
       car_fw = list(cached_params.carFw)

From 38cb3ccb8c2440ef62258f7c089b14ac97571d92 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 15:41:51 -0700
Subject: [PATCH 25/29] log to make sure it's taking exactly timeout time

---
 selfdrive/car/car_helpers.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 10f9348b520762..1049c87d96f619 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -12,6 +12,7 @@
 from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint
+from common.realtime import sec_since_boot
 
 EventName = car.CarEvent.EventName
 
@@ -98,7 +99,9 @@ def fingerprint(logcan, sendcan):
     else:
       cloudlog.warning("Getting VIN & FW versions")
       _, vin = get_vin(logcan, sendcan, bus)
+      ecu_t = sec_since_boot()
       ecu_responses = get_present_ecus(logcan, sendcan, get_interface_attr('FW_VERSIONS', ignore_none=True))
+      ecu_t = sec_since_boot() - ecu_t
       car_fw = get_fw_versions(logcan, sendcan)
 
     exact_fw_match, fw_candidates = match_fw_to_car(car_fw)
@@ -166,7 +169,7 @@ def fingerprint(logcan, sendcan):
     source = car.CarParams.FingerprintSource.fixed
 
   cloudlog.event("fingerprinted", car_fingerprint=car_fingerprint,
-                 source=source, fuzzy=not exact_match, fw_count=len(car_fw), ecu_responses=ecu_responses)
+                 source=source, fuzzy=not exact_match, fw_count=len(car_fw), ecu_responses=ecu_responses, ecu_t=ecu_t)
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
 

From d911ba3da534cde4bffd116d600532e52dbe0094 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 16:21:31 -0700
Subject: [PATCH 26/29] import time

---
 selfdrive/car/car_helpers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 1049c87d96f619..830ab311c57537 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -15,6 +15,7 @@
 from common.realtime import sec_since_boot
 
 EventName = car.CarEvent.EventName
+VERSIONS = get_interface_attr('FW_VERSIONS', ignore_none=True)
 
 
 def get_startup_event(car_recognized, controller_available, fw_seen):
@@ -100,7 +101,7 @@ def fingerprint(logcan, sendcan):
       cloudlog.warning("Getting VIN & FW versions")
       _, vin = get_vin(logcan, sendcan, bus)
       ecu_t = sec_since_boot()
-      ecu_responses = get_present_ecus(logcan, sendcan, get_interface_attr('FW_VERSIONS', ignore_none=True))
+      ecu_responses = get_present_ecus(logcan, sendcan, VERSIONS)
       ecu_t = sec_since_boot() - ecu_t
       car_fw = get_fw_versions(logcan, sendcan)
 

From b459f0bc3d28e44d4ed0ff7f059899472c80d805 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 18:29:43 -0700
Subject: [PATCH 27/29] fix logging

---
 selfdrive/car/fw_versions.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index 4970b4d801e425..b3d4479ae7761e 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -322,7 +322,8 @@ def get_present_ecus(logcan, sendcan, versions):
               queries.append([a])
 
           # Build set of expected responses to filter
-          responses.add((addr + r.rx_offset, sub_addr, r.bus))
+          response_addr = uds.get_rx_addr_for_tx_addr(addr, r.rx_offset)
+          responses.add((response_addr, sub_addr, r.bus))
 
   queries.insert(0, parallel_queries)
 
@@ -383,7 +384,7 @@ def get_fw_versions(logcan, sendcan, extra=None, timeout=0.1, debug=False, progr
     f.ecu = ecu_types[addr]
     f.fwVersion = version
     f.address = addr[0]
-    f.responseAddress = addr[0] + rx_offset
+    f.responseAddress = uds.get_rx_addr_for_tx_addr(addr[0], rx_offset)
     f.request = request
 
     if addr[1] is not None:

From fc5150e27cddec982eb2acb45541cc84b0befa59 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 19:55:02 -0700
Subject: [PATCH 28/29] 0.1 timeout

---
 selfdrive/car/car_helpers.py | 7 +++----
 selfdrive/car/fw_versions.py | 6 ++++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index 830ab311c57537..eec794be98e252 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -15,7 +15,6 @@
 from common.realtime import sec_since_boot
 
 EventName = car.CarEvent.EventName
-VERSIONS = get_interface_attr('FW_VERSIONS', ignore_none=True)
 
 
 def get_startup_event(car_recognized, controller_available, fw_seen):
@@ -101,7 +100,7 @@ def fingerprint(logcan, sendcan):
       cloudlog.warning("Getting VIN & FW versions")
       _, vin = get_vin(logcan, sendcan, bus)
       ecu_t = sec_since_boot()
-      ecu_responses = get_present_ecus(logcan, sendcan, VERSIONS)
+      ecu_responses = get_present_ecus(logcan, sendcan)
       ecu_t = sec_since_boot() - ecu_t
       car_fw = get_fw_versions(logcan, sendcan)
 
@@ -169,8 +168,8 @@ def fingerprint(logcan, sendcan):
     car_fingerprint = fixed_fingerprint
     source = car.CarParams.FingerprintSource.fixed
 
-  cloudlog.event("fingerprinted", car_fingerprint=car_fingerprint,
-                 source=source, fuzzy=not exact_match, fw_count=len(car_fw), ecu_responses=ecu_responses, ecu_t=ecu_t)
+  cloudlog.event("fingerprinted", car_fingerprint=car_fingerprint, source=source, fuzzy=not exact_match,
+                 fw_count=len(car_fw), ecu_responses=ecu_responses, ecu_t=ecu_t, error=True)
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
 
diff --git a/selfdrive/car/fw_versions.py b/selfdrive/car/fw_versions.py
index b3d4479ae7761e..66f0564fca6fda 100755
--- a/selfdrive/car/fw_versions.py
+++ b/selfdrive/car/fw_versions.py
@@ -300,10 +300,12 @@ def match_fw_to_car(fw_versions, allow_fuzzy=True):
   return exact_match, matches
 
 
-def get_present_ecus(logcan, sendcan, versions):
+def get_present_ecus(logcan, sendcan):
   queries = list()
   parallel_queries = list()
   responses = set()
+  versions = get_interface_attr('FW_VERSIONS', ignore_none=True)
+
   for r in REQUESTS:
     if r.brand not in versions:
       continue
@@ -329,7 +331,7 @@ def get_present_ecus(logcan, sendcan, versions):
 
   ecu_responses: Set[Tuple[int, Optional[int], int]] = set()
   for query in queries:
-    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses, timeout=0.2))
+    ecu_responses.update(get_ecu_addrs(logcan, sendcan, set(query), responses, timeout=0.1))
   return ecu_responses
 
 

From 7d6423dd27e5dca6ba291766b355e6fe809778b5 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Mon, 20 Jun 2022 21:46:33 -0700
Subject: [PATCH 29/29] clean up

---
 selfdrive/car/car_helpers.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py
index eec794be98e252..8c0fd7c90d6cdd 100644
--- a/selfdrive/car/car_helpers.py
+++ b/selfdrive/car/car_helpers.py
@@ -12,7 +12,6 @@
 from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint
-from common.realtime import sec_since_boot
 
 EventName = car.CarEvent.EventName
 
@@ -92,16 +91,14 @@ def fingerprint(logcan, sendcan):
       if cached_params.carName == "mock":
         cached_params = None
 
-    if False:  # cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
+    if cached_params is not None and len(cached_params.carFw) > 0 and cached_params.carVin is not VIN_UNKNOWN:
       cloudlog.warning("Using cached CarParams")
       vin = cached_params.carVin
       car_fw = list(cached_params.carFw)
     else:
       cloudlog.warning("Getting VIN & FW versions")
       _, vin = get_vin(logcan, sendcan, bus)
-      ecu_t = sec_since_boot()
       ecu_responses = get_present_ecus(logcan, sendcan)
-      ecu_t = sec_since_boot() - ecu_t
       car_fw = get_fw_versions(logcan, sendcan)
 
     exact_fw_match, fw_candidates = match_fw_to_car(car_fw)
@@ -169,7 +166,7 @@ def fingerprint(logcan, sendcan):
     source = car.CarParams.FingerprintSource.fixed
 
   cloudlog.event("fingerprinted", car_fingerprint=car_fingerprint, source=source, fuzzy=not exact_match,
-                 fw_count=len(car_fw), ecu_responses=ecu_responses, ecu_t=ecu_t, error=True)
+                 fw_count=len(car_fw), ecu_responses=ecu_responses, error=True)
   return car_fingerprint, finger, vin, car_fw, source, exact_match