diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index 1eda9bc7f59c89..363035ab224ec6 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -347,7 +347,7 @@ def replay_process(cfg, lr, fingerprint=None): else: return cpp_replay_process(cfg, lr, fingerprint) -def setup_env(simulation=False): +def setup_env(simulation=False, CP=None): params = Params() params.clear_all() params.put_bool("OpenpilotEnabledToggle", True) @@ -358,12 +358,22 @@ def setup_env(simulation=False): os.environ["NO_RADAR_SLEEP"] = "1" os.environ["REPLAY"] = "1" + os.environ['SKIP_FW_QUERY'] = "" + os.environ['FINGERPRINT'] = "" if simulation: os.environ["SIMULATION"] = "1" elif "SIMULATION" in os.environ: del os.environ["SIMULATION"] + # Regen or python process + if CP is not None: + if CP.fingerprintSource == "fw" and CP.carFingerprint in FW_VERSIONS: + params.put("CarParamsCache", CP.as_builder().to_bytes()) + else: + os.environ['SKIP_FW_QUERY'] = "1" + os.environ['FINGERPRINT'] = CP.carFingerprint + def python_replay_process(cfg, lr, fingerprint=None): sub_sockets = [s for _, sub in cfg.pub_sub.items() for s in sub] pub_sockets = [s for s in cfg.pub_sub.keys() if s != 'can'] @@ -378,30 +388,13 @@ def python_replay_process(cfg, lr, fingerprint=None): all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime) pub_msgs = [msg for msg in all_msgs if msg.which() in list(cfg.pub_sub.keys())] - setup_env() - - # TODO: remove after getting new route for civic & accord - migration = { - "HONDA CIVIC 2016 TOURING": "HONDA CIVIC 2016", - "HONDA ACCORD 2018 SPORT 2T": "HONDA ACCORD 2018", - "HONDA ACCORD 2T 2018": "HONDA ACCORD 2018", - "Mazda CX-9 2021": "MAZDA CX-9 2021", - } - if fingerprint is not None: os.environ['SKIP_FW_QUERY'] = "1" os.environ['FINGERPRINT'] = fingerprint + setup_env() else: - os.environ['SKIP_FW_QUERY'] = "" - os.environ['FINGERPRINT'] = "" - for msg in lr: - if msg.which() == 'carParams': - car_fingerprint = migration.get(msg.carParams.carFingerprint, msg.carParams.carFingerprint) - if msg.carParams.fingerprintSource == "fw" and (car_fingerprint in FW_VERSIONS): - Params().put("CarParamsCache", msg.carParams.as_builder().to_bytes()) - else: - os.environ['SKIP_FW_QUERY'] = "1" - os.environ['FINGERPRINT'] = car_fingerprint + CP = [m for m in lr if m.which() == 'carParams'][0].carParams + setup_env(CP=CP) assert(type(managed_processes[cfg.proc_name]) is PythonProcess) managed_processes[cfg.proc_name].prepare() diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index 653efaf32cfee2..dfc62a4558e0b9 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -15,7 +15,6 @@ from common.params import Params from common.realtime import Ratekeeper, DT_MDL, DT_DMON, sec_since_boot from common.transformations.camera import eon_f_frame_size, eon_d_frame_size, tici_f_frame_size, tici_d_frame_size -from selfdrive.car.fingerprints import FW_VERSIONS from selfdrive.manager.process import ensure_running from selfdrive.manager.process_config import managed_processes from selfdrive.test.process_replay.process_replay import FAKEDATA, setup_env, check_enabled @@ -181,28 +180,12 @@ def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False): if frs is None: frs = dict() - setup_env() params = Params() - os.environ["LOG_ROOT"] = outdir - os.environ['SKIP_FW_QUERY'] = "" - os.environ['FINGERPRINT'] = "" - - # TODO: remove after getting new route for Mazda - fp_migration = { - "Mazda CX-9 2021": "MAZDA CX-9 2021", - } - # TODO: remove after getting new route for Subaru - fingerprint_problem = ["SUBARU IMPREZA LIMITED 2019"] for msg in lr: if msg.which() == 'carParams': - car_fingerprint = fp_migration.get(msg.carParams.carFingerprint, msg.carParams.carFingerprint) - if len(msg.carParams.carFw) and (car_fingerprint in FW_VERSIONS) and (car_fingerprint not in fingerprint_problem): - params.put("CarParamsCache", msg.carParams.as_builder().to_bytes()) - else: - os.environ['SKIP_FW_QUERY'] = "1" - os.environ['FINGERPRINT'] = car_fingerprint + setup_env(CP=msg.carParams) elif msg.which() == 'liveCalibration': params.put("CalibrationParams", msg.as_builder().to_bytes()) diff --git a/selfdrive/test/process_replay/regen_all.py b/selfdrive/test/process_replay/regen_all.py index 765e5c3b682c90..c3ea1d41e1a8b9 100755 --- a/selfdrive/test/process_replay/regen_all.py +++ b/selfdrive/test/process_replay/regen_all.py @@ -8,26 +8,28 @@ from selfdrive.test.process_replay.helpers import OpenpilotPrefix from selfdrive.test.process_replay.regen import regen_and_save from selfdrive.test.process_replay.test_processes import FAKEDATA, original_segments as segments +from tools.lib.route import SegmentName -def regen_job(segment): + +def regen_job(segment, disable_tqdm): with OpenpilotPrefix(): - route = segment[1].rsplit('--', 1)[0] - sidx = int(segment[1].rsplit('--', 1)[1]) - fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for i in range(11)) + sn = SegmentName(segment[1]) + fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for _ in range(11)) try: - relr = regen_and_save(route, sidx, upload=True, use_route_meta=False, outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=True) + relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=True, use_route_meta=False, outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm) relr = '|'.join(relr.split('/')[-2:]) return f' ("{segment[0]}", "{relr}"), ' except Exception as e: return f" {segment} failed: {str(e)}" + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Generate new segments from old ones") parser.add_argument("-j", "--jobs", type=int, default=1) args = parser.parse_args() with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool: - p = list(pool.map(regen_job, segments)) + p = list(pool.map(regen_job, segments, [args.jobs > 1] * args.jobs)) msg = "Copy these new segments into test_processes.py:" for seg in tqdm(p, desc="Generating segments"): msg += "\n" + str(seg) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 9f83a08e23355b..25fbd210cc0a72 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -93,8 +93,7 @@ def test_process(cfg, lr, ref_log_path, ignore_fields=None, ignore_msgs=None): # check to make sure openpilot is engaged in the route if cfg.proc_name == "controlsd": if not check_enabled(log_msgs): - segment = os.path.basename(ref_log_path).split("/")[-1].split("_")[0] - raise Exception(f"Route never enabled: {segment}") + raise Exception(f"Route never enabled: {ref_log_path}") try: return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs